<commit_before># Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
# See LICENSE in the project root for license information.

from iris.constants import EMAIL_SUPPORT, IM_SUPPORT
from smtplib import SMTP
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import quopri
import time
import markdown
import dns.resolver
import dns.exception
import logging

logger = logging.getLogger()

class iris_smtp(object):
    supports = frozenset([EMAIL_SUPPORT, IM_SUPPORT])

    def __init__(self, config):
        self.config = config
        self.modes = {
            EMAIL_SUPPORT: self.send_email,
            IM_SUPPORT: self.send_email,
        }
        self.mx_sorted = []

        self.smtp_timeout = config.get('timeout', 10)
        if config.get('smtp_server'):
            # mock mx record
            self.mx_sorted.append((0, config['smtp_server']))
        elif config.get('smtp_gateway'):
            try:
                mx_hosts = dns.resolver.query(config['smtp_gateway'], 'MX')
            except dns.exception.DNSException as e:
                mx_hosts = []
                raise Exception('MX error: %s' % e)
            for mx in mx_hosts:
                mx_hostname = mx.exchange.to_text().strip('.')
                self.mx_sorted.append((mx.preference, mx_hostname))
            self.mx_sorted.sort(key=lambda tup: tup[0])
        else:
            raise ValueError('Missing SMTP config for sender')

    def send_email(self, message):
        md = markdown.Markdown()
        from_address = self.config['from']

        start = time.time()
        m = MIMEMultipart('alternative')
        m['from'] = from_address
        m['to'] = message['destination']
        if message.get('noreply'):
            m['reply-to'] = m['to']

        if 'email_subject' in message:
            m['subject'] = message['email_subject']
        else:
            m['subject'] = message['subject']

        plaintext = None

        if 'email_text' in message:
            plaintext = message['email_text']
        elif 'body' in message:
            plaintext = message['body']

        if plaintext:
            mt = MIMEText(None, 'plain', 'utf-8')
            mt.set_payload(quopri.encodestring(plaintext.encode('UTF-8')))
            mt.replace_header('Content-Transfer-Encoding', 'quoted-printable')
            m.attach(mt)

        # for tracking messages, email_html is not required, so it's possible
        # that both of the following keys are missing from message
        html = None

        if 'email_html' in message:
            html = message['email_html']
        elif 'body' in message:
            html = md.convert(message['body'])

        if html:
            if 'extra_html' in message:
                html += message['extra_html']
            # We need to have body tags for the oneclick buttons to properly parse
            html = '\n' + html + '\n'
            mt = MIMEText(None, 'html', 'utf-8')
            # Google does not like base64 encoded emails for the oneclick button functionalty,
            # so force quoted printable.
            mt.set_payload(quopri.encodestring(html.encode('UTF-8')))
            mt.replace_header('Content-Transfer-Encoding', 'quoted-printable')
            m.attach(mt)

        conn = None

        for mx in self.mx_sorted:
            try:
                smtp = SMTP(timeout=self.smtp_timeout)
                smtp.connect(mx[1], 25)
                conn = smtp
                break
            except Exception as e:
                logger.exception(e)

        if not conn:
            raise Exception('Failed to get smtp connection.')
        conn.sendmail([from_address], [message['destination']], m.as_string())
        conn.quit()

        return time.time() - start

    def send(self, message):
        return self.modes[message['mode']](message)
<commit_msg>Reuse smtp handle and avoid making one per message<commit_after># Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
# See LICENSE in the project root for license information.

from iris.constants import EMAIL_SUPPORT, IM_SUPPORT
from smtplib import SMTP
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import quopri
import time
import markdown
import dns.resolver
import dns.exception
import logging

logger = logging.getLogger()

class iris_smtp(object):
    supports = frozenset([EMAIL_SUPPORT, IM_SUPPORT])

    def __init__(self, config):
        self.config = config
        self.modes = {
            EMAIL_SUPPORT: self.send_email,
            IM_SUPPORT: self.send_email,
        }
        self.mx_sorted = []
        self.smtp_handles = {}  # map mailserver -> connection handle

        self.smtp_timeout = config.get('timeout', 10)
        if config.get('smtp_server'):
            # mock mx record
            self.mx_sorted.append((0, config['smtp_server']))
        elif config.get('smtp_gateway'):
            try:
                mx_hosts = dns.resolver.query(config['smtp_gateway'], 'MX')
            except dns.exception.DNSException as e:
                mx_hosts = []
                raise Exception('MX error: %s' % e)
            for mx in mx_hosts:
                mx_hostname = mx.exchange.to_text().strip('.')
                self.mx_sorted.append((mx.preference, mx_hostname))
            self.mx_sorted.sort(key=lambda tup: tup[0])
        else:
            raise ValueError('Missing SMTP config for sender')

    def send_email(self, message):
        md = markdown.Markdown()
        from_address = self.config['from']

        start = time.time()
        m = MIMEMultipart('alternative')
        m['from'] = from_address
        m['to'] = message['destination']
        if message.get('noreply'):
            m['reply-to'] = m['to']

        if 'email_subject' in message:
            m['subject'] = message['email_subject']
        else:
            m['subject'] = message['subject']

        plaintext = None

        if 'email_text' in message:
            plaintext = message['email_text']
        elif 'body' in message:
            plaintext = message['body']

        if plaintext:
            mt = MIMEText(None, 'plain', 'utf-8')
            mt.set_payload(quopri.encodestring(plaintext.encode('UTF-8')))
            mt.replace_header('Content-Transfer-Encoding', 'quoted-printable')
            m.attach(mt)

        # for tracking messages, email_html is not required, so it's possible
        # that both of the following keys are missing from message
        html = None

        if 'email_html' in message:
            html = message['email_html']
        elif 'body' in message:
            html = md.convert(message['body'])

        if html:
            if 'extra_html' in message:
                html += message['extra_html']
            # We need to have body tags for the oneclick buttons to properly parse
            html = '\n' + html + '\n'
            mt = MIMEText(None, 'html', 'utf-8')
            # Google does not like base64 encoded emails for the oneclick button functionalty,
            # so force quoted printable.
            mt.set_payload(quopri.encodestring(html.encode('UTF-8')))
            mt.replace_header('Content-Transfer-Encoding', 'quoted-printable')
            m.attach(mt)

        conn = None
        used_mx = None

        for mx in self.mx_sorted:
            old_handle = self.smtp_handles.get(mx)

            # Use previously created connection
            if old_handle:
                conn = old_handle
                used_mx = mx
                break

            # Otherwise try making a new one and storing it
            else:
                logger.info('Opening new MX connection for %s', mx)
                try:
                    smtp = SMTP(timeout=self.smtp_timeout)
                    smtp.connect(mx[1], 25)
                    conn = smtp
                    self.smtp_handles[mx] = conn
                    used_mx = mx
                    break
                except Exception as e:
                    logger.exception('Failed connecting to %s: %s', mx, e)

        if not conn:
            raise Exception('Failed to get smtp connection.')

        try:
            conn.sendmail([from_address], [message['destination']], m.as_string())
        except Exception as e:
            logger.exception('Failed sending email through %s: %s', used_mx, e)

            # When we fail, remove this handle from our map so we try connecting again next round
            self.smtp_handles.pop(used_mx, None)

            try:
                conn.quit()
            except Exception:
                logger.exception('Failed closing SMTP connection')

            return None

        return time.time() - start

    def send(self, message):
        return self.modes[message['mode']](message)
<|endoftext|>


<commit_before>import time
import socket
import logging
import hashlib
from datetime import datetime, timedelta

import requests
from requests import codes
from django.conf import settings
from django.utils.timezone import utc
from django.utils.encoding import force_text, force_bytes

import feedparser
import pytz
from mailme.core import exc
from mailme.core.models import (
    Feed,
    Post,
    Category,
    Enclosure,
    ACCEPTED_STATUSES,
    FEED_GENERIC_ERROR_TEXT,
    FEED_TIMEDOUT_ERROR_TEXT,
    FEED_NOT_FOUND_ERROR_TEXT
)

logger = logging.getLogger(__name__)

GUID_FIELDS = frozenset(("title", "link", "author"))

def find_post_content(feed_obj, entry):
    try:
        content = entry["content"][0]["value"]
    except (IndexError, KeyError):
        content = entry.get("description") or entry.get("summary") or ""
    return content

def date_to_datetime(field_name):
    """Given a post field, convert its :mod:`feedparser` date tuple to
    :class:`datetime.datetime` objects.

    :param field_name: The post field to use.

    """

    def _field_to_datetime(feed_obj, entry):
        if field_name in entry:
            try:
                time_ = time.mktime(entry[field_name])
                date = datetime.fromtimestamp(time_).replace(tzinfo=utc)
            except TypeError:
                date = datetime.now(pytz.utc)
            return date
        return datetime.now(pytz.utc)
    _field_to_datetime.__doc__ = "Convert %s to datetime" % repr(field_name)

    return _field_to_datetime

def generate_guid(entry):
    """Generate missing guid for post entry."""
    text = "|".join(force_text(entry.get(key) or "") for key in GUID_FIELDS)
    return hashlib.md5(force_bytes(text)).hexdigest()

def get_entry_guid(feed_obj, entry):
    """Get the guid for a post.

    If the post doesn't have a guid, a new guid is generated.

    """
    if "guid" not in entry:
        return generate_guid(entry)

    guid = force_text(entry["guid"]).strip()
    return guid

def format_date(t):
    """Make sure time object is a :class:`datetime.datetime` object."""
    if isinstance(t, time.struct_time):
        return datetime(*t[:6], tzinfo=pytz.utc)
    return t.replace(tzinfo=utc)

def entries_by_date(entries, limit=None):
    """Sort the feed entries by date

    :param entries: Entries given from :mod:`feedparser``.
    :param limit: Limit number of posts.

    """
    now = datetime.now(pytz.utc)

    def find_date(entry, counter):
        """Find the most current date entry tuple."""

        return (entry.get("published_parsed") or
                entry.get("date_parsed") or
                now - timedelta(seconds=(counter * 30)))

    sorted_entries = []
    for counter, entry in enumerate(entries):
        date = format_date(find_date(entry, counter))
        # the found date is put into the entry
        # because some feed just don't have any valid dates.
        # This will ensure that the posts will be properly ordered
        # later on when put into the database.
        entry["published_parsed"] = (entry.get("published_parsed")
                                     or date.timetuple())
        sorted_entries.append((date, entry))

    sorted_entries.sort(key=lambda key: key[0])
    sorted_entries.reverse()
    return [entry for _date, entry in sorted_entries[:limit]]

class FeedImporter(object):
    """Import/Update feeds.

    :keyword post_limit: See :attr`post_limit`.
    :keyword update_on_import: See :attr:`update_on_import`.
    :keyword logger: See :attr:`logger`.
    :keyword timeout: See :attr:`timeout`.

    .. attribute:: post_limit

        Default number of posts limit.

    .. attribute:: update_on_import

        By default, fetch new posts when a feed is imported

    .. attribute:: logger

       The :class:`logging.Logger` instance used for logging messages.

    .. attribute:: include_categories

        By default, include feed/post categories.

    .. attribute:: include_enclosures

        By default, include post enclosures.

    .. attribute:: timeout

        Default feed timeout.

    .. attribute:: parser

        The feed parser used. (Default: :mod:`feedparser`.)

    """
    update_on_import = True
    post_field_handlers = {
        "content": find_post_content,
        "date_published": date_to_datetime("published_parsed"),
        "date_updated": date_to_datetime("published_parsed"),
        "link": lambda feed_obj, entry: entry.get("link") or feed_obj.feed_url,
        "feed": lambda feed_obj, entry: feed_obj,
        "guid": get_entry_guid,
        "title": lambda feed_obj, entry: entry.get("title", "(no title)").strip(),
        "author": lambda feed_obj, entry: entry.get("author", "").strip(),
    }

    def __init__(self, **kwargs):
        self.post_limit = kwargs.get("post_limit", settings.POST_LIMIT)
        self.update_on_import = kwargs.get("update_on_import", True)
        self.include_categories = kwargs.get("include_categories", True)
        self.include_enclosures = kwargs.get("include_enclosures", True)
        self.timeout = kwargs.get("timeout", settings.FEED_TIMEOUT)

    def parse_feed(self, feed_url, etag=None, modified=None, timeout=None,
            maxlen=None):
        """Parse feed using feedparser.

        :param feed_url: URL to the feed to parse.

        :keyword etag: E-tag recevied from last parse (if any).
        :keyword modified: ``Last-Modified`` HTTP header received from last
            parse (if any).
        :keyword timeout: Parser timeout in seconds.

        """
        prev_timeout = socket.getdefaulttimeout()
        timeout = timeout or self.timeout
        socket.setdefaulttimeout(timeout)

        try:
            if maxlen:
                headers = requests.head(feed_url).headers
                contentlen = int(headers.get("content-length") or 0)
                if contentlen > maxlen:
                    raise exc.FeedCriticalError(FEED_GENERIC_ERROR_TEXT)

            feed = feedparser.parse(feed_url,
                                    etag=etag,
                                    modified=modified)
        finally:
            socket.setdefaulttimeout(prev_timeout)

        return feed

    def import_feed(self, feed_url, force=None, local=False):
        """Import feed.

        If feed is not seen before it will be created, otherwise
        just updated.

        :param feed_url: URL to the feed to import.
        :keyword force: Force import of feed even if it's been updated
            recently.
        """
        feed_url = feed_url.strip()
        feed = None
        try:
            feed_obj = Feed.objects.get(feed_url=feed_url)
        except Feed.DoesNotExist:
            try:
                feed = self.parse_feed(feed_url)
            except socket.timeout:
                Feed.objects.create(feed_url=feed_url)
                raise exc.TimeoutError(FEED_TIMEDOUT_ERROR_TEXT)
            except Exception:
                feed = {"status": 500}

            default_status = codes.NOT_FOUND
            if local:
                default_status = codes.OK

            status = feed.get("status", default_status)
            if status == codes.NOT_FOUND:
                raise exc.FeedNotFoundError(FEED_NOT_FOUND_ERROR_TEXT)
            if status not in ACCEPTED_STATUSES:
                raise exc.FeedCriticalError(
                    FEED_GENERIC_ERROR_TEXT,
                    status=status)

            # Feed can be local/fetched with a HTTP client.
            status = feed.get("status") or feed.get("status\n") or codes.OK

            if status == codes.FOUND or status == codes.MOVED_PERMANENTLY:
                if feed_url != feed.href:
                    return self.import_feed(feed.href, force=force)

            feed_name = feed.channel.get("title", "(no title)").strip()

            feed_obj = Feed.objects.update_or_create(feed_url=feed_url, **{
                'name': feed_name,
                'description': feed.channel.get('description', '')
            })

            if self.include_categories:
                feed_obj.categories.add(*self.get_categories(feed.channel))

        if self.update_on_import:
            feed_obj = self.update_feed(feed_obj, feed=feed, force=force)

        return feed_obj

    def get_categories(self, obj):
        """Get and save categories."""
        return [self.create_category(*cat)
                    for cat in getattr(obj, "categories", [])]

    def create_category(self, domain, name):
        """Create new category.

        :param domain: The category domain.
        :param name: The name of the category.

        """
        return Category.objects.update_or_create(
                      name=name.strip(),
                      domain=domain and domain.strip() or "")

    def update_feed(self, feed_obj, feed=None, force=False):
        """Update (refresh) feed.

        The feed must already exist in the system, if not you have
        to import it using :meth:`import_feed`.

        :param feed_obj: the Feed object
        :keyword feed: If feed has already been parsed you can pass the
            structure returned by the parser so it doesn't have to be parsed
            twice.
        :keyword force: Force refresh of the feed even if it has been
            recently refreshed already.

        """
        now = datetime.utcnow().replace(tzinfo=utc)
        already_fresh = (feed_obj.date_last_refresh and
                         now < feed_obj.date_last_refresh +
                         settings.MIN_REFRESH_INTERVAL)

        if already_fresh and not force:
            logger.info(
                "Feed %s is fresh. Skipping refresh." % feed_obj.feed_url)
            return feed_obj

        limit = self.post_limit
        if not feed:
            last_modified = None
            if feed_obj.http_last_modified and not force:
                last_modified = feed_obj.http_last_modified.timetuple()
            etag = feed_obj.http_etag if not force else None

            try:
                feed = self.parse_feed(feed_obj.feed_url,
                                       etag=etag,
                                       modified=last_modified)
            except socket.timeout:
                return feed_obj.save_timeout_error()
            except Exception:
                return feed_obj.save_generic_error()

        # Feed can be local/ not fetched with HTTP client.
        status = feed.get("status", codes.OK)
        if status == codes.NOT_MODIFIED and not force:
            return feed_obj

        if feed_obj.is_error_status(status):
            return feed_obj.set_error_status(status)

        if feed.entries:
            sorted_by_date = entries_by_date(feed.entries, limit)
            for entry in sorted_by_date:
                self.import_entry(entry, feed_obj)

        feed_obj.date_last_refresh = now
        feed_obj.http_etag = feed.get("etag", "")
        if hasattr(feed, "modified") and feed.modified:
            try:
                timestamp = time.mktime(feed.modified)
                modified = datetime.fromtimestamp(timestamp).replace(tzinfo=utc)
                feed_obj.http_last_modified = modified
            except TypeError:
                pass

        logger.debug("Saving feed object... %s" % (
                          feed_obj.feed_url))

        feed_obj.save()
        return feed_obj

    def create_enclosure(self, **kwargs):
        """Create new enclosure."""
        kwargs.setdefault("length", 0)
        return Enclosure.objects.update_or_create(**kwargs)

    def get_enclosures(self, entry):
        """Get and create enclosures for feed."""
        return [self.create_enclosure(url=enclosure.href,
                                      length=enclosure.length,
                                      type=enclosure.type)
                    for enclosure in getattr(entry, "enclosures", [])
                        if enclosure and hasattr(enclosure, "length")]

    def post_fields_parsed(self, entry, feed_obj):
        """Parse post fields."""
        return dict((key, handler(feed_obj, entry))
                        for key, handler in list(self.post_field_handlers.items()))

    def import_entry(self, entry, feed_obj):
        """Import feed post entry."""
        logger.debug("Importing entry... %s" % feed_obj.feed_url)

        fields = self.post_fields_parsed(entry, feed_obj)
        post = Post.objects.update_or_create(feed_obj, **fields)

        if self.include_enclosures:
            post.enclosures.add(*(self.get_enclosures(entry) or []))
        if self.include_categories:
            post.categories.add(*(self.get_categories(entry) or []))

        logger.debug("Post successfully imported... %s" % (
            feed_obj.feed_url))

        return post
<commit_msg>Sanitize remote html a bit, e.g remove deprecated tags<commit_after>import time
import socket
import logging
import hashlib
from datetime import datetime, timedelta

import requests
from requests import codes
from django.conf import settings
from django.utils.timezone import utc
from django.utils.encoding import force_text, force_bytes

import feedparser
import pytz
from mailme.core import exc
from mailme.core.models import (
    Feed,
    Post,
    Category,
    Enclosure,
    ACCEPTED_STATUSES,
    FEED_GENERIC_ERROR_TEXT,
    FEED_TIMEDOUT_ERROR_TEXT,
    FEED_NOT_FOUND_ERROR_TEXT
)
from mailme.utils.html import cleanup_html

logger = logging.getLogger(__name__)

GUID_FIELDS = frozenset(("title", "link", "author"))

def find_post_content(feed_obj, entry):
    try:
        content = entry["content"][0]["value"]
    except (IndexError, KeyError):
        content = entry.get("description") or entry.get("summary") or ""
    return cleanup_html(content)

def date_to_datetime(field_name):
    """Given a post field, convert its :mod:`feedparser` date tuple to
    :class:`datetime.datetime` objects.

    :param field_name: The post field to use.

    """

    def _field_to_datetime(feed_obj, entry):
        if field_name in entry:
            try:
                time_ = time.mktime(entry[field_name])
                date = datetime.fromtimestamp(time_).replace(tzinfo=utc)
            except TypeError:
                date = datetime.now(pytz.utc)
            return date
        return datetime.now(pytz.utc)
    _field_to_datetime.__doc__ = "Convert %s to datetime" % repr(field_name)

    return _field_to_datetime

def generate_guid(entry):
    """Generate missing guid for post entry."""
    text = "|".join(force_text(entry.get(key) or "") for key in GUID_FIELDS)
    return hashlib.md5(force_bytes(text)).hexdigest()

def get_entry_guid(feed_obj, entry):
    """Get the guid for a post.

    If the post doesn't have a guid, a new guid is generated.

    """
    if "guid" not in entry:
        return generate_guid(entry)

    guid = force_text(entry["guid"]).strip()
    return guid

def format_date(t):
    """Make sure time object is a :class:`datetime.datetime` object."""
    if isinstance(t, time.struct_time):
        return datetime(*t[:6], tzinfo=pytz.utc)
    return t.replace(tzinfo=utc)

def entries_by_date(entries, limit=None):
    """Sort the feed entries by date

    :param entries: Entries given from :mod:`feedparser``.
    :param limit: Limit number of posts.

    """
    now = datetime.now(pytz.utc)

    def find_date(entry, counter):
        """Find the most current date entry tuple."""

        return (entry.get("published_parsed") or
                entry.get("date_parsed") or
                now - timedelta(seconds=(counter * 30)))

    sorted_entries = []
    for counter, entry in enumerate(entries):
        date = format_date(find_date(entry, counter))
        # the found date is put into the entry
        # because some feed just don't have any valid dates.
        # This will ensure that the posts will be properly ordered
        # later on when put into the database.
        entry["published_parsed"] = (entry.get("published_parsed")
                                     or date.timetuple())
        sorted_entries.append((date, entry))

    sorted_entries.sort(key=lambda key: key[0])
    sorted_entries.reverse()
    return [entry for _date, entry in sorted_entries[:limit]]

class FeedImporter(object):
    """Import/Update feeds.

    :keyword post_limit: See :attr`post_limit`.
    :keyword update_on_import: See :attr:`update_on_import`.
    :keyword logger: See :attr:`logger`.
    :keyword timeout: See :attr:`timeout`.

    .. attribute:: post_limit

        Default number of posts limit.

    .. attribute:: update_on_import

        By default, fetch new posts when a feed is imported

    .. attribute:: logger

       The :class:`logging.Logger` instance used for logging messages.

    .. attribute:: include_categories

        By default, include feed/post categories.

    .. attribute:: include_enclosures

        By default, include post enclosures.

    .. attribute:: timeout

        Default feed timeout.

    .. attribute:: parser

        The feed parser used. (Default: :mod:`feedparser`.)

    """
    update_on_import = True
    post_field_handlers = {
        "content": find_post_content,
        "date_published": date_to_datetime("published_parsed"),
        "date_updated": date_to_datetime("published_parsed"),
        "link": lambda feed_obj, entry: entry.get("link") or feed_obj.feed_url,
        "feed": lambda feed_obj, entry: feed_obj,
        "guid": get_entry_guid,
        "title": lambda feed_obj, entry: entry.get("title", "(no title)").strip(),
        "author": lambda feed_obj, entry: entry.get("author", "").strip(),
    }

    def __init__(self, **kwargs):
        self.post_limit = kwargs.get("post_limit", settings.POST_LIMIT)
        self.update_on_import = kwargs.get("update_on_import", True)
        self.include_categories = kwargs.get("include_categories", True)
        self.include_enclosures = kwargs.get("include_enclosures", True)
        self.timeout = kwargs.get("timeout", settings.FEED_TIMEOUT)

    def parse_feed(self, feed_url, etag=None, modified=None, timeout=None,
            maxlen=None):
        """Parse feed using feedparser.

        :param feed_url: URL to the feed to parse.

        :keyword etag: E-tag recevied from last parse (if any).
        :keyword modified: ``Last-Modified`` HTTP header received from last
            parse (if any).
        :keyword timeout: Parser timeout in seconds.

        """
        prev_timeout = socket.getdefaulttimeout()
        timeout = timeout or self.timeout
        socket.setdefaulttimeout(timeout)

        try:
            if maxlen:
                headers = requests.head(feed_url).headers
                contentlen = int(headers.get("content-length") or 0)
                if contentlen > maxlen:
                    raise exc.FeedCriticalError(FEED_GENERIC_ERROR_TEXT)

            feed = feedparser.parse(feed_url,
                                    etag=etag,
                                    modified=modified)
        finally:
            socket.setdefaulttimeout(prev_timeout)

        return feed

    def import_feed(self, feed_url, force=None, local=False):
        """Import feed.

        If feed is not seen before it will be created, otherwise
        just updated.

        :param feed_url: URL to the feed to import.
        :keyword force: Force import of feed even if it's been updated
            recently.
        """
        feed_url = feed_url.strip()
        feed = None
        try:
            feed_obj = Feed.objects.get(feed_url=feed_url)
        except Feed.DoesNotExist:
            try:
                feed = self.parse_feed(feed_url)
            except socket.timeout:
                Feed.objects.create(feed_url=feed_url)
                raise exc.TimeoutError(FEED_TIMEDOUT_ERROR_TEXT)
            except Exception:
                feed = {"status": 500}

            default_status = codes.NOT_FOUND
            if local:
                default_status = codes.OK

            status = feed.get("status", default_status)
            if status == codes.NOT_FOUND:
                raise exc.FeedNotFoundError(FEED_NOT_FOUND_ERROR_TEXT)
            if status not in ACCEPTED_STATUSES:
                raise exc.FeedCriticalError(
                    FEED_GENERIC_ERROR_TEXT,
                    status=status)

            # Feed can be local/fetched with a HTTP client.
            status = feed.get("status") or feed.get("status\n") or codes.OK

            if status == codes.FOUND or status == codes.MOVED_PERMANENTLY:
                if feed_url != feed.href:
                    return self.import_feed(feed.href, force=force)

            feed_name = feed.channel.get("title", "(no title)").strip()

            feed_obj = Feed.objects.update_or_create(feed_url=feed_url, **{
                'name': feed_name,
                'description': feed.channel.get('description', '')
            })

            if self.include_categories:
                feed_obj.categories.add(*self.get_categories(feed.channel))

        if self.update_on_import:
            feed_obj = self.update_feed(feed_obj, feed=feed, force=force)

        return feed_obj

    def get_categories(self, obj):
        """Get and save categories."""
        return [self.create_category(*cat)
                    for cat in getattr(obj, "categories", [])]

    def create_category(self, domain, name):
        """Create new category.

        :param domain: The category domain.
        :param name: The name of the category.

        """
        return Category.objects.update_or_create(
                      name=name.strip(),
                      domain=domain and domain.strip() or "")

    def update_feed(self, feed_obj, feed=None, force=False):
        """Update (refresh) feed.

        The feed must already exist in the system, if not you have
        to import it using :meth:`import_feed`.

        :param feed_obj: the Feed object
        :keyword feed: If feed has already been parsed you can pass the
            structure returned by the parser so it doesn't have to be parsed
            twice.
        :keyword force: Force refresh of the feed even if it has been
            recently refreshed already.

        """
        now = datetime.utcnow().replace(tzinfo=utc)
        already_fresh = (feed_obj.date_last_refresh and
                         now < feed_obj.date_last_refresh +
                         settings.MIN_REFRESH_INTERVAL)

        if already_fresh and not force:
            logger.info(
                "Feed %s is fresh. Skipping refresh." % feed_obj.feed_url)
            return feed_obj

        limit = self.post_limit
        if not feed:
            last_modified = None
            if feed_obj.http_last_modified and not force:
                last_modified = feed_obj.http_last_modified.timetuple()
            etag = feed_obj.http_etag if not force else None

            try:
                feed = self.parse_feed(feed_obj.feed_url,
                                       etag=etag,
                                       modified=last_modified)
            except socket.timeout:
                return feed_obj.save_timeout_error()
            except Exception:
                return feed_obj.save_generic_error()

        # Feed can be local/ not fetched with HTTP client.
        status = feed.get("status", codes.OK)
        if status == codes.NOT_MODIFIED and not force:
            return feed_obj

        if feed_obj.is_error_status(status):
            return feed_obj.set_error_status(status)

        if feed.entries:
            sorted_by_date = entries_by_date(feed.entries, limit)
            for entry in sorted_by_date:
                self.import_entry(entry, feed_obj)

        feed_obj.date_last_refresh = now
        feed_obj.http_etag = feed.get("etag", "")
        if hasattr(feed, "modified") and feed.modified:
            try:
                timestamp = time.mktime(feed.modified)
                modified = datetime.fromtimestamp(timestamp).replace(tzinfo=utc)
                feed_obj.http_last_modified = modified
            except TypeError:
                pass

        logger.debug("Saving feed object... %s" % (
                          feed_obj.feed_url))

        feed_obj.save()
        return feed_obj

    def create_enclosure(self, **kwargs):
        """Create new enclosure."""
        kwargs.setdefault("length", 0)
        return Enclosure.objects.update_or_create(**kwargs)

    def get_enclosures(self, entry):
        """Get and create enclosures for feed."""
        return [self.create_enclosure(url=enclosure.href,
                                      length=enclosure.length,
                                      type=enclosure.type)
                    for enclosure in getattr(entry, "enclosures", [])
                        if enclosure and hasattr(enclosure, "length")]

    def post_fields_parsed(self, entry, feed_obj):
        """Parse post fields."""
        return dict((key, handler(feed_obj, entry))
                        for key, handler in list(self.post_field_handlers.items()))

    def import_entry(self, entry, feed_obj):
        """Import feed post entry."""
        logger.debug("Importing entry... %s" % feed_obj.feed_url)

        fields = self.post_fields_parsed(entry, feed_obj)
        post = Post.objects.update_or_create(feed_obj, **fields)

        if self.include_enclosures:
            post.enclosures.add(*(self.get_enclosures(entry) or []))
        if self.include_categories:
            post.categories.add(*(self.get_categories(entry) or []))

        logger.debug("Post successfully imported... %s" % (
            feed_obj.feed_url))

        return post
<|endoftext|>


<commit_before>import collections
import six
import re

from autotranslate.compat import goslate, googleapiclient

from django.conf import settings

from yandex_translate import YandexTranslate

class BaseTranslatorService:
    """
    Defines the base methods that should be implemented
    """

    def translate_string(self, text, target_language, source_language='en'):
        """
        Returns a single translated string literal for the target language.
        """
        raise NotImplementedError('.translate_string() must be overridden.')

    def translate_strings(self, strings, target_language, source_language='en', optimized=True):
        """
        Returns a iterator containing translated strings for the target language
        in the same order as in the strings.
        :return:    if `optimized` is True returns a generator else an array
        """
        raise NotImplementedError('.translate_strings() must be overridden.')

class GoSlateTranslatorService(BaseTranslatorService):
    """
    Uses the free web-based API for translating.
    https://bitbucket.org/zhuoqiang/goslate
    """

    def __init__(self):
        self.developer_key = getattr(settings, 'YANDEX_TRANSLATE_KEY', None)
        self.yandex_translate_obj = YandexTranslate(self.developer_key)

        # self.number = self.yandex_translate_obj.translate("_|_number_|_(~d~", 'fr')
        # self.text_item = self.yandex_translate_obj.translate("_|_item_|_(~s~", 'fr')
        # self.character_s = self.yandex_translate_obj.translate("s", 'fr')
        # self.character_d = self.yandex_translate_obj.translate("d", 'fr')

        # assert goslate, '`GoSlateTranslatorService` requires `goslate` package'
        # self.service = goslate.Goslate()

    def translate_string(self, text, target_language, source_language='en'):
        assert isinstance(text, six.string_types), '`text` should a string literal'
        direction = source_language+'-'+target_language
        response = self.yandex_translate_obj.translate(text, direction)
        return response['text'][0]

    def translate_strings(self, strings, target_language, source_language='en', optimized=True):
        assert isinstance(strings, collections.Iterable), '`strings` should a iterable containing string_types'
        direction = source_language+'-'+target_language
        translation_list = strings
        count = 0

        self.number = self.yandex_translate_obj.translate("__number__(~d~", direction)
        self.text_item = self.yandex_translate_obj.translate("__item__(~s~", direction)
        self.character_s = self.yandex_translate_obj.translate("s", direction)
        self.character_d = self.yandex_translate_obj.translate("d", direction)

        from autotranslate.utils import look_placeholders
        from .management.commands.translate_messages import fix_translation

        import time
        time.sleep(15) # delays for 15 seconds

        for item in strings:
            variable = None
            response = self.yandex_translate_obj.translate(item, direction, 'html')
            translation_response = response['text'][0]
            try:
                translation_response = fix_translation(item,  translation_response)
            except IndexError:
                pass

            if self.number['text'][0] in translation_response:
                translation_response = translation_response.replace(self.number['text'][0], "%d")

            if self.text_item['text'][0] in translation_response:
                translation_response = translation_response.replace(self.text_item['text'][0], "%s")

            if "__item__(~s~" in translation_response:
                translation_response = translation_response.replace('_|_item_|_(~s~', '%s')

            if "__number__(~d~" in translation_response:
                translation_response = translation_response.replace('__number__(~d~', '%d')

            variables = re.findall('\%\((.*?)\)', item)
            translate_variables = re.findall('\%\((.*?)\)', translation_response)
            for variable in variables:
                translation_response = re.sub('\%\((.*?)\)', '%('+variable+')', translation_response)

            variables = re.findall('\_\|_(.*?)_\|\_', item)
            translate_variables = re.findall('\_\|_(.*?)_\|\_', translation_response)
            for translate_variable, variable in zip( translate_variables, variables):
                if look_placeholders(item, variable, translation_response) == 's':
                    translation_response = re.sub(r'_\|_' + re.escape(translate_variable) + r'_\|_', '%(' + variable + ')', translation_response )
                    translation_response = re.sub(r'\(~' + re.escape(self.character_s['text'][0]) + r'~', 's', translation_response, 1)
                    translation_response = re.sub(r'\(~s~', 's', translation_response, 1)
                elif look_placeholders(item, variable, translation_response) == 'd':
                    translation_response = re.sub(r'_\|_' + re.escape(translate_variable) + r'_\|_', '%(' + variable + ')', translation_response )
                    translation_response = re.sub(r'\(~' + re.escape(self.character_d['text'][0]) + r'~', 'd', translation_response, 1)
                    translation_response = re.sub(r'\(~d~', 'd', translation_response, 1)
                else:
                    translation_response = re.sub(r'_\|_' + re.escape(translate_variable) + r'_\|_', '%(' + variable + ')', translation_response )

            variables = re.findall('\{(.*?)\}', item)
            translate_variables = re.findall('\{(.*?)\}', translation_response)
            for translate_variable, variable in zip( translate_variables, variables):
                translation_response = re.sub(r'\{' + re.escape(translate_variable) + r'\}', '{' + variable + '}', translation_response )

            if ')_s' in translation_response:
                translation_response = translation_response.replace(')_s', 's')

            if ')_d' in translation_response:
                translation_response = translation_response.replace(')_d', 'd')

            translation_list[count] = translation_response
            count += 1
            print translation_response.encode('utf-8')
            # if self.count > 100:
            #     break
        return translation_list

    # def __init__(self):
    #     assert goslate, '`GoSlateTranslatorService` requires `goslate` package'
    #     self.service = goslate.Goslate()
    #
    # def translate_string(self, text, target_language, source_language='en'):
    #     assert isinstance(text, six.string_types), '`text` should a string literal'
    #     return self.service.translate(text, target_language, source_language)

    # def translate_strings(self, strings, target_language, source_language='en', optimized=True):
    #     assert isinstance(strings, collections.Iterable), '`strings` should a iterable containing string_types'
    #     translations = self.service.translate(strings, target_language, source_language)
    #     return translations if optimized else [_ for _ in translations]

class GoogleAPITranslatorService(BaseTranslatorService):
    """
    Uses the paid Google API for translating.
    https://github.com/google/google-api-python-client
    """

    def __init__(self, max_segments=128):
        assert googleapiclient, '`GoogleAPITranslatorService` requires `google-api-python-client` package'

        self.developer_key = getattr(settings, 'GOOGLE_TRANSLATE_KEY', None)
        assert self.developer_key, ('`GOOGLE_TRANSLATE_KEY` is not configured, '
                                    'it is required by `GoogleAPITranslatorService`')

        from googleapiclient.discovery import build
        self.service = build('translate', 'v2', developerKey=self.developer_key)

        # the google translation API has a limit of max
        # 128 translations in a single request
        # and throws `Too many text segments Error`
        self.max_segments = max_segments
        self.translated_strings = []

    def translate_string(self, text, target_language, source_language='en'):
        assert isinstance(text, six.string_types), '`text` should a string literal'
        response = self.service.translations() \
            .list(source=source_language, target=target_language, q=[text]).execute()
        return response.get('translations').pop(0).get('translatedText')

    def translate_strings(self, strings, target_language, source_language='en', optimized=True):
        assert isinstance(strings, collections.MutableSequence), \
            '`strings` should be a sequence containing string_types'
        assert not optimized, 'optimized=True is not supported in `GoogleAPITranslatorService`'
        if len(strings) <= self.max_segments:
            setattr(self, 'translated_strings', getattr(self, 'translated_strings', []))
            response = self.service.translations() \
                .list(source=source_language, target=target_language, q=strings).execute()
            self.translated_strings.extend([t.get('translatedText') for t in response.get('translations')])
            return self.translated_strings
        else:
            self.translate_strings(strings[0:self.max_segments], target_language, source_language, optimized)
            _translated_strings = self.translate_strings(strings[self.max_segments:],
                                                         target_language, source_language, optimized)

            # reset the property or it will grow with subsequent calls
            self.translated_strings = []
            return _translated_strings
<commit_msg>removed delay - way too high<commit_after>import collections
import six
import re

from autotranslate.compat import goslate, googleapiclient

from django.conf import settings

from yandex_translate import YandexTranslate

class BaseTranslatorService:
    """
    Defines the base methods that should be implemented
    """

    def translate_string(self, text, target_language, source_language='en'):
        """
        Returns a single translated string literal for the target language.
        """
        raise NotImplementedError('.translate_string() must be overridden.')

    def translate_strings(self, strings, target_language, source_language='en', optimized=True):
        """
        Returns a iterator containing translated strings for the target language
        in the same order as in the strings.
        :return:    if `optimized` is True returns a generator else an array
        """
        raise NotImplementedError('.translate_strings() must be overridden.')

class GoSlateTranslatorService(BaseTranslatorService):
    """
    Uses the free web-based API for translating.
    https://bitbucket.org/zhuoqiang/goslate
    """

    def __init__(self):
        self.developer_key = getattr(settings, 'YANDEX_TRANSLATE_KEY', None)
        self.yandex_translate_obj = YandexTranslate(self.developer_key)

        # self.number = self.yandex_translate_obj.translate("_|_number_|_(~d~", 'fr')
        # self.text_item = self.yandex_translate_obj.translate("_|_item_|_(~s~", 'fr')
        # self.character_s = self.yandex_translate_obj.translate("s", 'fr')
        # self.character_d = self.yandex_translate_obj.translate("d", 'fr')

        # assert goslate, '`GoSlateTranslatorService` requires `goslate` package'
        # self.service = goslate.Goslate()

    def translate_string(self, text, target_language, source_language='en'):
        assert isinstance(text, six.string_types), '`text` should a string literal'
        direction = source_language+'-'+target_language
        response = self.yandex_translate_obj.translate(text, direction)
        return response['text'][0]

    def translate_strings(self, strings, target_language, source_language='en', optimized=True):
        assert isinstance(strings, collections.Iterable), '`strings` should a iterable containing string_types'
        direction = source_language+'-'+target_language
        translation_list = strings
        count = 0

        self.number = self.yandex_translate_obj.translate("__number__(~d~", direction)
        self.text_item = self.yandex_translate_obj.translate("__item__(~s~", direction)
        self.character_s = self.yandex_translate_obj.translate("s", direction)
        self.character_d = self.yandex_translate_obj.translate("d", direction)

        from autotranslate.utils import look_placeholders
        from .management.commands.translate_messages import fix_translation

        #import time
        #time.sleep(15) # delays for 15 seconds

        for item in strings:
            variable = None
            response = self.yandex_translate_obj.translate(item, direction, 'html')
            translation_response = response['text'][0]
            try:
                translation_response = fix_translation(item,  translation_response)
            except IndexError:
                pass

            if self.number['text'][0] in translation_response:
                translation_response = translation_response.replace(self.number['text'][0], "%d")

            if self.text_item['text'][0] in translation_response:
                translation_response = translation_response.replace(self.text_item['text'][0], "%s")

            if "__item__(~s~" in translation_response:
                translation_response = translation_response.replace('_|_item_|_(~s~', '%s')

            if "__number__(~d~" in translation_response:
                translation_response = translation_response.replace('__number__(~d~', '%d')

            variables = re.findall('\%\((.*?)\)', item)
            translate_variables = re.findall('\%\((.*?)\)', translation_response)
            for variable in variables:
                translation_response = re.sub('\%\((.*?)\)', '%('+variable+')', translation_response)

            variables = re.findall('\_\|_(.*?)_\|\_', item)
            translate_variables = re.findall('\_\|_(.*?)_\|\_', translation_response)
            for translate_variable, variable in zip( translate_variables, variables):
                if look_placeholders(item, variable, translation_response) == 's':
                    translation_response = re.sub(r'_\|_' + re.escape(translate_variable) + r'_\|_', '%(' + variable + ')', translation_response )
                    translation_response = re.sub(r'\(~' + re.escape(self.character_s['text'][0]) + r'~', 's', translation_response, 1)
                    translation_response = re.sub(r'\(~s~', 's', translation_response, 1)
                elif look_placeholders(item, variable, translation_response) == 'd':
                    translation_response = re.sub(r'_\|_' + re.escape(translate_variable) + r'_\|_', '%(' + variable + ')', translation_response )
                    translation_response = re.sub(r'\(~' + re.escape(self.character_d['text'][0]) + r'~', 'd', translation_response, 1)
                    translation_response = re.sub(r'\(~d~', 'd', translation_response, 1)
                else:
                    translation_response = re.sub(r'_\|_' + re.escape(translate_variable) + r'_\|_', '%(' + variable + ')', translation_response )

            variables = re.findall('\{(.*?)\}', item)
            translate_variables = re.findall('\{(.*?)\}', translation_response)
            for translate_variable, variable in zip( translate_variables, variables):
                translation_response = re.sub(r'\{' + re.escape(translate_variable) + r'\}', '{' + variable + '}', translation_response )

            if ')_s' in translation_response:
                translation_response = translation_response.replace(')_s', 's')

            if ')_d' in translation_response:
                translation_response = translation_response.replace(')_d', 'd')

            translation_list[count] = translation_response
            count += 1
            print translation_response.encode('utf-8')
            # if self.count > 100:
            #     break
        return translation_list

    # def __init__(self):
    #     assert goslate, '`GoSlateTranslatorService` requires `goslate` package'
    #     self.service = goslate.Goslate()
    #
    # def translate_string(self, text, target_language, source_language='en'):
    #     assert isinstance(text, six.string_types), '`text` should a string literal'
    #     return self.service.translate(text, target_language, source_language)

    # def translate_strings(self, strings, target_language, source_language='en', optimized=True):
    #     assert isinstance(strings, collections.Iterable), '`strings` should a iterable containing string_types'
    #     translations = self.service.translate(strings, target_language, source_language)
    #     return translations if optimized else [_ for _ in translations]

class GoogleAPITranslatorService(BaseTranslatorService):
    """
    Uses the paid Google API for translating.
    https://github.com/google/google-api-python-client
    """

    def __init__(self, max_segments=128):
        assert googleapiclient, '`GoogleAPITranslatorService` requires `google-api-python-client` package'

        self.developer_key = getattr(settings, 'GOOGLE_TRANSLATE_KEY', None)
        assert self.developer_key, ('`GOOGLE_TRANSLATE_KEY` is not configured, '
                                    'it is required by `GoogleAPITranslatorService`')

        from googleapiclient.discovery import build
        self.service = build('translate', 'v2', developerKey=self.developer_key)

        # the google translation API has a limit of max
        # 128 translations in a single request
        # and throws `Too many text segments Error`
        self.max_segments = max_segments
        self.translated_strings = []

    def translate_string(self, text, target_language, source_language='en'):
        assert isinstance(text, six.string_types), '`text` should a string literal'
        response = self.service.translations() \
            .list(source=source_language, target=target_language, q=[text]).execute()
        return response.get('translations').pop(0).get('translatedText')

    def translate_strings(self, strings, target_language, source_language='en', optimized=True):
        assert isinstance(strings, collections.MutableSequence), \
            '`strings` should be a sequence containing string_types'
        assert not optimized, 'optimized=True is not supported in `GoogleAPITranslatorService`'
        if len(strings) <= self.max_segments:
            setattr(self, 'translated_strings', getattr(self, 'translated_strings', []))
            response = self.service.translations() \
                .list(source=source_language, target=target_language, q=strings).execute()
            self.translated_strings.extend([t.get('translatedText') for t in response.get('translations')])
            return self.translated_strings
        else:
            self.translate_strings(strings[0:self.max_segments], target_language, source_language, optimized)
            _translated_strings = self.translate_strings(strings[self.max_segments:],
                                                         target_language, source_language, optimized)

            # reset the property or it will grow with subsequent calls
            self.translated_strings = []
            return _translated_strings
<|endoftext|>


<commit_before>import traceback

from submissions.Sery import aids

class DataFrame:
    data = []
    feature_names = []
    target = []
    target_names = []

aidsECHP = DataFrame()
aidsECHP.data = []
target_data = []

list_of_report = aids.get_reports()
for record in list_of_report:
    try:
        prevalence = float(record['Data']["HIV Prevalence"]["Adults"])
        target_data.append(prevalence)

        year = int(record['Year'])
        living =  int(record['Data']["People Living with HIV"]["Adults"])
        new = int(record['Data']["New HIV Infections"]["Adults"])
        deaths = int(record['Data']["AIDS-Related Deaths"]["Adults"])

        aidsECHP.data.append([year, living, new, deaths])

    except:
        traceback.print_exc()

aidsECHP.feature_names = [
    'Year',
    'People Living with HIV',
    'New HIV Infections',
    'AIDS-Related Deaths',
]

'''
Build the target list,
one entry for each row in the input frame.

The Naive Bayesian network is a classifier,
i.e. it sorts data points into bins.
The best it can do to estimate a continuous variable
is to break the domain into segments, and predict
the segment into which the variable's value will fall.
In this example, I'm breaking Trump's % into two
arbitrary segments.
'''
aidsECHP.target = []

def aidsTarget(percentage):
    if percentage > 4:
        return 1
    return 0

for pre in target_data:
    # choose the target
    tt = aidsTarget(pre)
    aidsECHP.target.append(tt)

aidsECHP.target_names = [
    'HIV Prevalence <= 4%',
    'HIV Prevalence > 4%',
]

Examples = {
    'Aids': aidsECHP,
}<commit_msg>updated percentage value<commit_after>import traceback

from submissions.Sery import aids

class DataFrame:
    data = []
    feature_names = []
    target = []
    target_names = []

aidsECHP = DataFrame()
aidsECHP.data = []
target_data = []

list_of_report = aids.get_reports()
for record in list_of_report:
    try:
        prevalence = float(record['Data']["HIV Prevalence"]["Adults"])
        target_data.append(prevalence)

        year = int(record['Year'])
        living =  int(record['Data']["People Living with HIV"]["Adults"])
        new = int(record['Data']["New HIV Infections"]["Adults"])
        deaths = int(record['Data']["AIDS-Related Deaths"]["Adults"])

        aidsECHP.data.append([year, living, new, deaths])

    except:
        traceback.print_exc()

aidsECHP.feature_names = [
    'Year',
    'People Living with HIV',
    'New HIV Infections',
    'AIDS-Related Deaths',
]

'''
Build the target list,
one entry for each row in the input frame.

The Naive Bayesian network is a classifier,
i.e. it sorts data points into bins.
The best it can do to estimate a continuous variable
is to break the domain into segments, and predict
the segment into which the variable's value will fall.
In this example, I'm breaking Trump's % into two
arbitrary segments.
'''
aidsECHP.target = []

def aidsTarget(percentage):
    if percentage > 6:
        return 1
    return 0

for pre in target_data:
    # choose the target
    tt = aidsTarget(pre)
    aidsECHP.target.append(tt)

aidsECHP.target_names = [
    'HIV Prevalence <= 6%',
    'HIV Prevalence > 6%',
]

Examples = {
    'Aids': aidsECHP,
}<|endoftext|>


<commit_before>import jsonfield
from copy import deepcopy
from datetime import datetime
from django.db import models
from django.contrib.contenttypes import generic
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from .managers import RevisionManager
from .utils import get_object_data

class Revision(models.Model):
    # Generic foreign key to the object
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField(db_index=True)
    content_object = generic.GenericForeignKey()

    # The serialized data for this revision
    data = jsonfield.JSONField(null=True, blank=True)

    # The user that created the revision
    user = models.ForeignKey(User, null=True, blank=True,
                             related_name='+revision')
    session_key = models.CharField(max_length=40, null=True, blank=True)

    # The timestamp of the revision
    timestamp = models.DateTimeField(default=datetime.now, db_index=True)

    # This revision represents a deleted object
    deleted = models.BooleanField(default=False)

    # The changes from the previous revision
    changes = jsonfield.JSONField(null=True, blank=True)

    objects = RevisionManager()

    class Meta(object):
        app_label = 'avocado'
        ordering = ('-timestamp',)
        get_latest_by = 'timestamp'

    def __unicode__(self):
        return '{0} @ {1}'.format(self.content_object, self.timestamp)

    def apply(self, instance, commit=True):
        "Applies this revision to an instance."
        for key, value in self.data.items():
            setattr(instance, key, deepcopy(value))
        if commit:
            instance.save()
        return instance

    def diff(self, instance, fields=None, reverse=False):
        """
        Returns the diff between the revision and the instance, relative
        to the revision. Changed values will include the old and new values
        under the 'old_value' and 'new_value' keys respectively.

        If `reverse` is true, the diff will be relative to the instance.
        If `fields` is not defined, the revision data keys will be used.
        """
        diff = {}

        if not fields:
            fields = self.data.keys()

        data = get_object_data(instance, fields=fields)

        for key in fields:
            if data.get(key) != self.data.get(key):
                if reverse:
                    old_value = data.get(key)
                    new_value = self.data.get(key)
                else:
                    old_value = self.data.get(key)
                    new_value = data.get(key)
                diff[key] = {
                    'old_value': old_value,
                    'new_value': new_value
                }
        return diff
<commit_msg>Handle the case when revision data is null<commit_after>import jsonfield
from copy import deepcopy
from datetime import datetime
from django.db import models
from django.contrib.contenttypes import generic
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from .managers import RevisionManager
from .utils import get_object_data

class Revision(models.Model):
    # Generic foreign key to the object
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField(db_index=True)
    content_object = generic.GenericForeignKey()

    # The serialized data for this revision
    data = jsonfield.JSONField(null=True, blank=True)

    # The user that created the revision
    user = models.ForeignKey(User, null=True, blank=True,
                             related_name='+revision')
    session_key = models.CharField(max_length=40, null=True, blank=True)

    # The timestamp of the revision
    timestamp = models.DateTimeField(default=datetime.now, db_index=True)

    # This revision represents a deleted object
    deleted = models.BooleanField(default=False)

    # The changes from the previous revision
    changes = jsonfield.JSONField(null=True, blank=True)

    objects = RevisionManager()

    class Meta(object):
        app_label = 'avocado'
        ordering = ('-timestamp',)
        get_latest_by = 'timestamp'

    def __unicode__(self):
        return '{0} @ {1}'.format(self.content_object, self.timestamp)

    def apply(self, instance, commit=True):
        "Applies this revision to an instance."
        for key, value in self.data.items():
            setattr(instance, key, deepcopy(value))
        if commit:
            instance.save()
        return instance

    def diff(self, instance, fields=None, reverse=False):
        """
        Returns the diff between the revision and the instance, relative
        to the revision. Changed values will include the old and new values
        under the 'old_value' and 'new_value' keys respectively.

        If `reverse` is true, the diff will be relative to the instance.
        If `fields` is not defined, the revision data keys will be used.
        """
        diff = {}

        current = get_object_data(instance, fields=fields)
        previous = self.data or {}

        if not fields:
            fields = previous.keys()

        for key in fields:
            if current.get(key) != previous.get(key):
                if reverse:
                    old_value = current.get(key)
                    new_value = previous.get(key)
                else:
                    old_value = previous.get(key)
                    new_value = current.get(key)

                diff[key] = {
                    'old_value': old_value,
                    'new_value': new_value
                }

        return diff
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging

__all__ = []

log = logging.getLogger(__name__)

class NoLanguage(object):
    """ This class is the context when no language nor nodes are specified."""
    pass

def get_root_resource(request):

    log.debug('get_root_resource: %s', request.path_info)

    url_parts = [url_part
                 for url_part in request.path_info.strip('/').split('/')
                 if url_part]

    log.debug('url_parts: %s', url_parts)

    if not url_parts:
        # URL is '/'.
        log.debug('Return NoLanguage context.')
        return NoLanguage()

    lang = url_parts.pop(0)

    if not url_parts:
        # URL is like '/{lang}'.
        #criterion = Language.lang.ilike(lang)
        #query = request.db_session.query(Language).filter(criterion)
        return {lang: lang}#query.first()}

    resource = {}
    resource[lang] = {}

    # for user, address in session.query(User, adalias).join(adalias, User.addresses):
    queries = []
    pivot = resource[lang]

    for url_part in url_parts:

        pivot[url_part] = {}
        pivot = pivot[url_part]

        #criterion = and_(NodeInfo.url_part.ilike(url_part),
        #                 NodeInfo.lang.has(NodeInfo.lang.ilike(lang)))
        #query = request.db_session.query(NodeInfo).filter(criterion)
        #NodeInfoAlias = aliased(NodeInfo, query.subquery())
        #query = request.db_session.query(Node).join(NodeInfoAlias, Node.translations)

    log.debug(resource)

    if url_parts:
        nodes = url_parts

    return resource
<commit_msg>Completed 'The Smart Traversal': the function 'get_root_resource' build a resource tree without any query. This version is based on new 'url' argument in NodeInfo model.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from aybu.website.models.language import Language
from aybu.website.models.node import NodeInfo
import logging

__all__ = []

log = logging.getLogger(__name__)

def get_root_resource(request):

    log.debug('get_root_resource: %s', request.path_info)

    url_parts = [(url_part, Resource())
                 for url_part in request.path_info.strip('/').split('/')
                 if url_part]

    log.debug('url_parts: %s', url_parts)

    if not url_parts:
        # URL is '/'.
        log.debug('Return NoLanguage context.')
        return NoLanguage()

    if len(url_parts) == 1:
        # URL is like '/{lang}'.
        log.debug('Return Language %s (if exists) or None.', lang)
        return Language.get_by_lang(request.db_session, url_parts[0])

    else:
        # URL is like '/{lang}/{node}/[...]/{page}[.ext]
        # Get the NodeInfo from database using path_info.
        url_parts[-1] = (url_parts[-1][0],
                         NodeInfo.get_by_url(request.db_session,
                                             request.path_info))

    # Create the resources tree.
    # The last element in resources tree is the request context.
    tmp = root = {}
    for url_part, resource in url_parts:
        tmp[url_part] = resource
        tmp = tmp[url_part]
        key = url_part

    log.debug('Resource is: %s', root)
    log.debug('Pivot is: %s', tmp)
    log.debug('Key is: %s', key)

    return root

class NoLanguage(object):
    """ This class is the context when no language nor nodes are specified."""
    pass

class Resource(dict):
    """
        This class is a basic resource.
        it is needed to build ACL, do NOT use for requests context!
    """
    pass
<|endoftext|>


<commit_before>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# Built-in Imports
import requests
from requests import Request,Session,Response
import json
import constants
import sys
import os
import auth
import utils
from resourcegroup import *
from cloudify.exceptions import NonRecoverableError
from cloudify import ctx
from cloudify.decorators import operation
import auth

@operation
def creation_validation(**_):
    for property_key in constants.PUBLIC_IP_REQUIRED_PROPERTIES:
        _validate_node_properties(property_key, ctx.node.properties)

    public_ip_exists =  _get_public_ip_name()

    if ctx.node.properties['use_external_resource'] and not public_ip_exists:
    	raise NonRecoverableError(
    	'External resource, but the supplied '
    	'public ip does not exist in the account.')
    if not ctx.node.properties['use_external_resource'] and public_ip_exists:
    	raise NonRecoverableError(
    	'Not external resource, but the supplied '
    	'public ip exists in the account.')

@operation
def create_public_ip(**_):
    if ctx.node.properties['use_external_resource']
        if not public_ip_exists:
	    	raise NonRecoverableError(
		'External resource, but the supplied '
		'public ip does not exist in the account.')
		sys.exit(1)
        else
        	ctx.instance.runtime_properties[constants.PUBLIC_IP_KEY] = ctx.node.properties['existing_public_ip_name']
    else

    subscription_id = ctx.node.properties['subscription_id']
    location = ctx.node.properties['location']
    resource_group_name = ctx.node.instance.runtime_properties['resource_group']
    RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
    public_ip_name=contants.PUBLIC_IP_PREFIX+RANDOM_SUFFIX_VALUE
    public_ip_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/publicIPAddresses/'+public_ip_name+'?api-version='+constants.api_version

    credentials='Bearer '+ auth.get_token_from_client_credentials()
    headers = {"Content-Type": "application/json", "Authorization": credentials}

    try:
        ctx.logger.info("Creating new public ip : " + public_ip_name)
        public_ip_params=json.dumps({
                "location": location,
                "name": public_ip_name,
                "properties": {
                    "publicIPAllocationMethod": "Dynamic",
                    "idleTimeoutInMinutes": 4,
                }
            }
        )
        response_pip = requests.put(url=public_ip_url, data=public_ip_params, headers=headers)
        print response_pip.text
        ctx.instance.runtime_properties['publicip']=public_ip_name

    except:
        ctx.logger.info("Public IP" + public_ip_name + "could not be created.")
        sys.exit(1)

def delete_public_ip(**_):

    subscription_id = ctx.node.properties['subscription_id']
    resource_group_name = ctx.node.instance.runtime_properties['resource_group']

    credentials='Bearer '+ auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    try:
        ctx.logger.info("Deleting Public IP")
        public_ip_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/ publicIPAddresses/'+public_ip_name+'?api-version='+constants.api_version
        response_pip = requests.delete(url=public_ip_url,headers=headers)
        print(response_pip.text)

    except:
        ctx.logger.info("Public IP " + public_ip_name + " could not be deleted.")
        sys.exit(1)

@operation
def set_dependent_resources_names(azure_config,**kwargs):
   ctx.source.instance.runtime_properties['resource_group'] = ctx.target.instance.runtime_properties['resource_group']

def _validate_node_properties(key, ctx_node_properties):
    if key not in ctx_node_properties:
        raise NonRecoverableError('{0} is a required input. Unable to create.'.format(key))

def _get_public_ip_name():
    public_ip_name=ctx.node.properties['existing_public_ip_name']
    resource_group_name = ctx.node.instance.runtime_properties['resource_group']
    credentials=auth.get_token_from_client_credentials()
    headers={"Content-Type": "application/json", "Authorization": credentials}
    subscription_id=ctx.node.properties['subscription_id']
    pip_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/publicIPAddresses?api-version='+constants.api_version
    response_get_pip=requests.get(url=pip_url,headers=headers)
    if public_ip_name in response_get_pip.text:
        return True
    else:
        return False
<commit_msg>Update publicip.py<commit_after>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# Built-in Imports
import requests
from requests import Request,Session,Response
import json
import constants
import sys
import os
import auth
import utils
from resourcegroup import *
from cloudify.exceptions import NonRecoverableError
from cloudify import ctx
from cloudify.decorators import operation
import auth

@operation
def creation_validation(**_):
    for property_key in constants.PUBLIC_IP_REQUIRED_PROPERTIES:
        _validate_node_properties(property_key, ctx.node.properties)

    public_ip_exists =  _get_public_ip_name()

    if ctx.node.properties['use_external_resource'] and not public_ip_exists:
    	raise NonRecoverableError(
    	'External resource, but the supplied '
    	'public ip does not exist in the account.')
    if not ctx.node.properties['use_external_resource'] and public_ip_exists:
    	raise NonRecoverableError(
    	'Not external resource, but the supplied '
    	'public ip exists in the account.')

@operation
def create_public_ip(**_):
    if ctx.node.properties['use_external_resource']
        if not public_ip_exists:
	    	raise NonRecoverableError(
		'External resource, but the supplied '
		'public ip does not exist in the account.')
		sys.exit(1)
        else
        	ctx.instance.runtime_properties[constants.PUBLIC_IP_KEY] = ctx.node.properties['existing_public_ip_name']
    else

    subscription_id = ctx.node.properties['subscription_id']
    location = ctx.node.properties['location']
    resource_group_name = ctx.instance.runtime_properties['resource_group']
    RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
    public_ip_name=contants.PUBLIC_IP_PREFIX+RANDOM_SUFFIX_VALUE
    public_ip_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/publicIPAddresses/'+public_ip_name+'?api-version='+constants.api_version

    credentials='Bearer '+ auth.get_token_from_client_credentials()
    headers = {"Content-Type": "application/json", "Authorization": credentials}

    try:
        ctx.logger.info("Creating new public ip : " + public_ip_name)
        public_ip_params=json.dumps({
                "location": location,
                "name": public_ip_name,
                "properties": {
                    "publicIPAllocationMethod": "Dynamic",
                    "idleTimeoutInMinutes": 4,
                }
            }
        )
        response_pip = requests.put(url=public_ip_url, data=public_ip_params, headers=headers)
        print response_pip.text
        ctx.instance.runtime_properties['publicip']=public_ip_name

    except:
        ctx.logger.info("Public IP" + public_ip_name + "could not be created.")
        sys.exit(1)

def delete_public_ip(**_):

    subscription_id = ctx.node.properties['subscription_id']
    resource_group_name = ctx.instance.runtime_properties['resource_group']

    credentials='Bearer '+ auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    try:
        ctx.logger.info("Deleting Public IP")
        public_ip_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/ publicIPAddresses/'+public_ip_name+'?api-version='+constants.api_version
        response_pip = requests.delete(url=public_ip_url,headers=headers)
        print(response_pip.text)

    except:
        ctx.logger.info("Public IP " + public_ip_name + " could not be deleted.")
        sys.exit(1)

@operation
def set_dependent_resources_names(azure_config,**kwargs):
   ctx.source.instance.runtime_properties['resource_group'] = ctx.target.instance.runtime_properties['resource_group']

def _validate_node_properties(key, ctx_node_properties):
    if key not in ctx_node_properties:
        raise NonRecoverableError('{0} is a required input. Unable to create.'.format(key))

def _get_public_ip_name():
    public_ip_name=ctx.node.properties['existing_public_ip_name']
    resource_group_name = ctx.instance.runtime_properties['resource_group']
    credentials=auth.get_token_from_client_credentials()
    headers={"Content-Type": "application/json", "Authorization": credentials}
    subscription_id=ctx.node.properties['subscription_id']
    pip_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/publicIPAddresses?api-version='+constants.api_version
    response_get_pip=requests.get(url=pip_url,headers=headers)
    if public_ip_name in response_get_pip.text:
        return True
    else:
        return False
<|endoftext|>


<commit_before>from django.test import TestCase
from django.core.urlresolvers import reverse

from .models import Routine, Exercise

class RoutineAPITests(TestCase):

    def setUp(self):
        Routine.objects.create(name='test1', total_time='0:05:00')
        Exercise.objects.create(name='warm up', completion_time='0:01:00', position=1, routine__name='test1')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=2, routine__name='test1')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=3, routine__name='test1')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=4, routine__name='test1')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=5, routine__name='test1')

        Routine.objects.create(name='test2', total_time='0:05:00')
        Exercise.objects.create(name='warm up', completion_time='0:01:00', position=1, routine__name='test2')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=2, routine__name='test2')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=3, routine__name='test2')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=4, routine__name='test2')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=5, routine__name='test2')

    def test_getRoutines(self):

        """
            All routines should be returned
        """

        routinesFromDb = Routine.objects.all()

        url = reverse('routines:get_routines')

        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)  # Make sure valid request returns success response

        self.assertEquals(routinesFromDb, response.data)  # Make sure routines from API match routines in DB

    def test_getRoutine(self):

        """
            A routine should be returned when a correct id is passed as a parameter
        """
        pass

    def test_createRoutines(self):
        pass

    def test_deleteRoutine(self):
        pass

    def test_createExercises(self):
        pass

    def test_createExercise(self):
        pass

    def test_deleteExercise(self):
        pass

 # def test_getFemale(self):

 #        """
 #            A female should be returned when a correct id is passed as a parameter
 #        """

 #        testFemale = Female.objects.all()[:1].get()
 #        testIdentifier = testFemale.id

 #        url = reverse('females:get_female')

 #        response = self.client.get(url)
 #        self.assertEquals(response.status_code, 400)  # Make sure no params return error response

 #        response = self.client.get(url, {'wrong': 'wrong'}, content_type='application/json')
 #        self.assertEquals(response.status_code, 400)  # Make sure bad params return error response

 #        response = self.client.get(url, {'identifier': testIdentifier})
 #        self.assertEquals(response.status_code, 200)  # Make sure valid request returns success response

 #        data = json.loads(response.content.decode())
 #        self.assertEquals(len(data), 6)  # Make sure all fields are present

 #        self.assertEquals(testIdentifier, data['id'])  # Make sure the correct female was returned
<commit_msg>Server - Added get routine test<commit_after>import json

from django.test import TestCase
from django.core.urlresolvers import reverse

from .models import Routine, Exercise

class RoutineAPITests(TestCase):

    def setUp(self):
        Routine.objects.create(name='test1', total_time='0:05:00')
        Exercise.objects.create(name='warm up', completion_time='0:01:00', position=1, routine__name='test1')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=2, routine__name='test1')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=3, routine__name='test1')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=4, routine__name='test1')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=5, routine__name='test1')

        Routine.objects.create(name='test2', total_time='0:05:00')
        Exercise.objects.create(name='warm up', completion_time='0:01:00', position=1, routine__name='test2')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=2, routine__name='test2')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=3, routine__name='test2')
        Exercise.objects.create(name='jog', completion_time='0:01:00', position=4, routine__name='test2')
        Exercise.objects.create(name='run', completion_time='0:01:00', position=5, routine__name='test2')

    def test_getRoutines(self):

        """
            All routines should be returned
        """

        routinesFromDb = Routine.objects.all()

        url = reverse('routines:get_routines')

        response = self.client.get(url)
        self.assertEquals(response.status_code, 200)  # Make sure valid request returns success response

        self.assertEquals(routinesFromDb, response.data)  # Make sure routines from API match routines in routinesFromDbS

    def test_getRoutine(self):

        """
            A routine and associated exercises should be returned when a correct id is passed as a parameter
        """

        routineFromDb = Routine.objects.all()[:1].get()

        url = reverse('routines:get_routine')

        response = self.client.post(url, {'wrong': 'wrong'})
        self.assertEquals(response.status_code, 400)  # Make sure bad params return error response

        response = self.client.get(url, {'id': routineFromDb.id})
        self.assertEquals(response.status_code, 200)  # Make sure valid request returns success response

        data = json.loads(response.content.decode())
        self.assertEquals(routineFromDb, data)  # Make sure the correct routine was returned

    def test_createRoutines(self):
        pass

    def test_deleteRoutine(self):
        pass

    def test_createExercises(self):
        pass

    def test_createExercise(self):
        pass

    def test_deleteExercise(self):
        pass

 # def test_getFemale(self):

 #        """
 #            A female should be returned when a correct id is passed as a parameter
 #        """

 #        testFemale = Female.objects.all()[:1].get()
 #        testIdentifier = testFemale.id

 #        url = reverse('females:get_female')

 #        response = self.client.get(url)
 #        self.assertEquals(response.status_code, 400)  # Make sure no params return error response

 #        response = self.client.get(url, {'wrong': 'wrong'}, content_type='application/json')
 #        self.assertEquals(response.status_code, 400)  # Make sure bad params return error response

 #        response = self.client.get(url, {'identifier': testIdentifier})
 #        self.assertEquals(response.status_code, 200)  # Make sure valid request returns success response

 #        data = json.loads(response.content.decode())
 #        self.assertEquals(len(data), 6)  # Make sure all fields are present

 #        self.assertEquals(testIdentifier, data['id'])  # Make sure the correct female was returned
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Electrum - lightweight Bitcoin client
# Copyright (C) 2011 thomasv@gitorious
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import hashlib, base64, re

def rev_hex(s):
    return s.decode('hex')[::-1].encode('hex')

def int_to_hex(i, length=1):
    s = hex(i)[2:].rstrip('L')
    s = "0"*(2*length - len(s)) + s
    return rev_hex(s)

def var_int(i):
    if i<0xfd:
        return int_to_hex(i)
    elif i<=0xffff:
        return "fd"+int_to_hex(i,2)
    elif i<=0xffffffff:
        return "fe"+int_to_hex(i,4)
    else:
        return "ff"+int_to_hex(i,8)

Hash = lambda x: hashlib.sha256(hashlib.sha256(x).digest()).digest()
hash_encode = lambda x: x[::-1].encode('hex')
hash_decode = lambda x: x.decode('hex')[::-1]

def header_to_string(res):
    pbh = res.get('prev_block_hash')
    if pbh is None: pbh = '0'*64
    s = int_to_hex(res.get('version'),4) \
        + rev_hex(pbh) \
        + rev_hex(res.get('merkle_root')) \
        + int_to_hex(int(res.get('timestamp')),4) \
        + int_to_hex(int(res.get('bits')),4) \
        + int_to_hex(int(res.get('nonce')),4)
    return s

def header_from_string( s):
    hex_to_int = lambda s: eval('0x' + s[::-1].encode('hex'))
    h = {}
    h['version'] = hex_to_int(s[0:4])
    h['prev_block_hash'] = hash_encode(s[4:36])
    h['merkle_root'] = hash_encode(s[36:68])
    h['timestamp'] = hex_to_int(s[68:72])
    h['bits'] = hex_to_int(s[72:76])
    h['nonce'] = hex_to_int(s[76:80])
    return h

############ functions from pywallet #####################

addrtype = 0

def hash_160(public_key):
    try:
        md = hashlib.new('ripemd160')
        md.update(hashlib.sha256(public_key).digest())
        return md.digest()
    except:
        import ripemd
        md = ripemd.new(hashlib.sha256(public_key).digest())
        return md.digest()

def public_key_to_bc_address(public_key):
    if public_key == 'None': return 'None'
    h160 = hash_160(public_key)
    return hash_160_to_bc_address(h160)

def hash_160_to_bc_address(h160):
    if h160 == 'None': return 'None'
    vh160 = chr(addrtype) + h160
    h = Hash(vh160)
    addr = vh160 + h[0:4]
    return b58encode(addr)

def bc_address_to_hash_160(addr):
    bytes = b58decode(addr, 25)
    return bytes[1:21]

__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
__b58base = len(__b58chars)

def b58encode(v):
    """ encode v, which is a string of bytes, to base58."""

    long_value = 0L
    for (i, c) in enumerate(v[::-1]):
        long_value += (256**i) * ord(c)

    result = ''
    while long_value >= __b58base:
        div, mod = divmod(long_value, __b58base)
        result = __b58chars[mod] + result
        long_value = div
    result = __b58chars[long_value] + result

    # Bitcoin does a little leading-zero-compression:
    # leading 0-bytes in the input become leading-1s
    nPad = 0
    for c in v:
        if c == '\0': nPad += 1
        else: break

    return (__b58chars[0]*nPad) + result

def b58decode(v, length):
    """ decode v into a string of len bytes."""
    long_value = 0L
    for (i, c) in enumerate(v[::-1]):
        long_value += __b58chars.find(c) * (__b58base**i)

    result = ''
    while long_value >= 256:
        div, mod = divmod(long_value, 256)
        result = chr(mod) + result
        long_value = div
    result = chr(long_value) + result

    nPad = 0
    for c in v:
        if c == __b58chars[0]: nPad += 1
        else: break

    result = chr(0)*nPad + result
    if length is not None and len(result) != length:
        return None

    return result

def EncodeBase58Check(vchIn):
    hash = Hash(vchIn)
    return b58encode(vchIn + hash[0:4])

def DecodeBase58Check(psz):
    vchRet = b58decode(psz, None)
    key = vchRet[0:-4]
    csum = vchRet[-4:]
    hash = Hash(key)
    cs32 = hash[0:4]
    if cs32 != csum:
        return None
    else:
        return key

def PrivKeyToSecret(privkey):
    return privkey[9:9+32]

def SecretToASecret(secret):
    vchIn = chr(addrtype+128) + secret
    return EncodeBase58Check(vchIn)

def ASecretToSecret(key):
    vch = DecodeBase58Check(key)
    if vch and vch[0] == chr(addrtype+128):
        return vch[1:]
    else:
        return False

########### end pywallet functions #######################

<commit_msg>fix: this wasnt the right place<commit_after>#!/usr/bin/env python
#
# Electrum - lightweight Bitcoin client
# Copyright (C) 2011 thomasv@gitorious
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import hashlib, base64, re

def rev_hex(s):
    return s.decode('hex')[::-1].encode('hex')

def int_to_hex(i, length=1):
    s = hex(i)[2:].rstrip('L')
    s = "0"*(2*length - len(s)) + s
    return rev_hex(s)

def var_int(i):
    if i<0xfd:
        return int_to_hex(i)
    elif i<=0xffff:
        return "fd"+int_to_hex(i,2)
    elif i<=0xffffffff:
        return "fe"+int_to_hex(i,4)
    else:
        return "ff"+int_to_hex(i,8)

Hash = lambda x: hashlib.sha256(hashlib.sha256(x).digest()).digest()
hash_encode = lambda x: x[::-1].encode('hex')
hash_decode = lambda x: x.decode('hex')[::-1]

def header_to_string(res):
    pbh = res.get('prev_block_hash')
    if pbh is None: pbh = '0'*64
    s = int_to_hex(res.get('version'),4) \
        + rev_hex(pbh) \
        + rev_hex(res.get('merkle_root')) \
        + int_to_hex(int(res.get('timestamp')),4) \
        + int_to_hex(int(res.get('bits')),4) \
        + int_to_hex(int(res.get('nonce')),4)
    return s

def header_from_string( s):
    hex_to_int = lambda s: eval('0x' + s[::-1].encode('hex'))
    h = {}
    h['version'] = hex_to_int(s[0:4])
    h['prev_block_hash'] = hash_encode(s[4:36])
    h['merkle_root'] = hash_encode(s[36:68])
    h['timestamp'] = hex_to_int(s[68:72])
    h['bits'] = hex_to_int(s[72:76])
    h['nonce'] = hex_to_int(s[76:80])
    return h

############ functions from pywallet #####################

addrtype = 0

def hash_160(public_key):
    try:
        md = hashlib.new('ripemd160')
        md.update(hashlib.sha256(public_key).digest())
        return md.digest()
    except:
        import ripemd
        md = ripemd.new(hashlib.sha256(public_key).digest())
        return md.digest()

def public_key_to_bc_address(public_key):
    h160 = hash_160(public_key)
    return hash_160_to_bc_address(h160)

def hash_160_to_bc_address(h160):
    if h160 == 'None': return 'None'
    vh160 = chr(addrtype) + h160
    h = Hash(vh160)
    addr = vh160 + h[0:4]
    return b58encode(addr)

def bc_address_to_hash_160(addr):
    if addr == 'None': return 'None'
    bytes = b58decode(addr, 25)
    return bytes[1:21]

__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
__b58base = len(__b58chars)

def b58encode(v):
    """ encode v, which is a string of bytes, to base58."""

    long_value = 0L
    for (i, c) in enumerate(v[::-1]):
        long_value += (256**i) * ord(c)

    result = ''
    while long_value >= __b58base:
        div, mod = divmod(long_value, __b58base)
        result = __b58chars[mod] + result
        long_value = div
    result = __b58chars[long_value] + result

    # Bitcoin does a little leading-zero-compression:
    # leading 0-bytes in the input become leading-1s
    nPad = 0
    for c in v:
        if c == '\0': nPad += 1
        else: break

    return (__b58chars[0]*nPad) + result

def b58decode(v, length):
    """ decode v into a string of len bytes."""
    long_value = 0L
    for (i, c) in enumerate(v[::-1]):
        long_value += __b58chars.find(c) * (__b58base**i)

    result = ''
    while long_value >= 256:
        div, mod = divmod(long_value, 256)
        result = chr(mod) + result
        long_value = div
    result = chr(long_value) + result

    nPad = 0
    for c in v:
        if c == __b58chars[0]: nPad += 1
        else: break

    result = chr(0)*nPad + result
    if length is not None and len(result) != length:
        return None

    return result

def EncodeBase58Check(vchIn):
    hash = Hash(vchIn)
    return b58encode(vchIn + hash[0:4])

def DecodeBase58Check(psz):
    vchRet = b58decode(psz, None)
    key = vchRet[0:-4]
    csum = vchRet[-4:]
    hash = Hash(key)
    cs32 = hash[0:4]
    if cs32 != csum:
        return None
    else:
        return key

def PrivKeyToSecret(privkey):
    return privkey[9:9+32]

def SecretToASecret(secret):
    vchIn = chr(addrtype+128) + secret
    return EncodeBase58Check(vchIn)

def ASecretToSecret(key):
    vch = DecodeBase58Check(key)
    if vch and vch[0] == chr(addrtype+128):
        return vch[1:]
    else:
        return False

########### end pywallet functions #######################

<|endoftext|>