<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals

from datetime import datetime
from dateutil import tz
import re

try:
    from functools import lru_cache
except ImportError:
    from backports.functools_lru_cache import lru_cache

from arrow import locales

class ParserError(RuntimeError):
    pass

class DateTimeParser(object):

    _FORMAT_RE = re.compile('(YYY?Y?|MM?M?M?|Do|DD?D?D?|d?d?d?d|HH?|hh?|mm?|ss?|S+|ZZ?Z?|a|A|X)')
    _ESCAPE_RE = re.compile('\[[^\[\]]*\]')

    _ONE_OR_MORE_DIGIT_RE = re.compile('\d+')
    _ONE_OR_TWO_DIGIT_RE = re.compile('\d{1,2}')
    _FOUR_DIGIT_RE = re.compile('\d{4}')
    _TWO_DIGIT_RE = re.compile('\d{2}')
    _TZ_RE = re.compile('[+\-]?\d{2}:?(\d{2})?')
    _TZ_NAME_RE = re.compile('\w[\w+\-/]+')

    _BASE_INPUT_RE_MAP = {
        'YYYY': _FOUR_DIGIT_RE,
        'YY': _TWO_DIGIT_RE,
        'MM': _TWO_DIGIT_RE,
        'M': _ONE_OR_TWO_DIGIT_RE,
        'DD': _TWO_DIGIT_RE,
        'D': _ONE_OR_TWO_DIGIT_RE,
        'HH': _TWO_DIGIT_RE,
        'H': _ONE_OR_TWO_DIGIT_RE,
        'hh': _TWO_DIGIT_RE,
        'h': _ONE_OR_TWO_DIGIT_RE,
        'mm': _TWO_DIGIT_RE,
        'm': _ONE_OR_TWO_DIGIT_RE,
        'ss': _TWO_DIGIT_RE,
        's': _ONE_OR_TWO_DIGIT_RE,
        'X': re.compile('\d+'),
        'ZZZ': _TZ_NAME_RE,
        'ZZ': _TZ_RE,
        'Z': _TZ_RE,
        'S': _ONE_OR_MORE_DIGIT_RE,
    }

    MARKERS = ['YYYY', 'MM', 'DD']
    SEPARATORS = ['-', '/', '.']

    def __init__(self, locale='en_us', cache_size=0):

        self.locale = locales.get_locale(locale)
        self._input_re_map = self._BASE_INPUT_RE_MAP.copy()
        self._input_re_map.update({
            'MMMM': self._choice_re(self.locale.month_names[1:], re.IGNORECASE),
            'MMM': self._choice_re(self.locale.month_abbreviations[1:],
                                   re.IGNORECASE),
            'Do': re.compile(self.locale.ordinal_day_re),
            'dddd': self._choice_re(self.locale.day_names[1:], re.IGNORECASE),
            'ddd': self._choice_re(self.locale.day_abbreviations[1:],
                                   re.IGNORECASE),
            'd': re.compile(r"[1-7]"),
            'a': self._choice_re(
                (self.locale.meridians['am'], self.locale.meridians['pm'])
            ),
            # note: 'A' token accepts both 'am/pm' and 'AM/PM' formats to
            # ensure backwards compatibility of this token
            'A': self._choice_re(self.locale.meridians.values())
        })
        if cache_size > 0:
            self._generate_pattern_re =\
                lru_cache(maxsize=cache_size)(self._generate_pattern_re)

    def parse_iso(self, string):

        has_time = 'T' in string or ' ' in string.strip()
        space_divider = ' ' in string.strip()

        if has_time:
            if space_divider:
                date_string, time_string = string.split(' ', 1)
            else:
                date_string, time_string = string.split('T', 1)
            time_parts = re.split('[+-]', time_string, 1)
            has_tz = len(time_parts) > 1
            has_seconds = time_parts[0].count(':') > 1
            has_subseconds = re.search('[.,]', time_parts[0])

            if has_subseconds:
                formats = ['YYYY-MM-DDTHH:mm:ss%sS' % has_subseconds.group()]
            elif has_seconds:
                formats = ['YYYY-MM-DDTHH:mm:ss']
            else:
                formats = ['YYYY-MM-DDTHH:mm']
        else:
            has_tz = False
            # generate required formats: YYYY-MM-DD, YYYY-MM-DD, YYYY
            # using various separators: -, /, .
            l = len(self.MARKERS)
            formats = [separator.join(self.MARKERS[:l-i])
                       for i in range(l)
                       for separator in self.SEPARATORS]

        if has_time and has_tz:
            formats = [f + 'Z' for f in formats]

        if space_divider:
            formats = [item.replace('T', ' ', 1) for item in formats]

        return self._parse_multiformat(string, formats)

    def _generate_pattern_re(self, fmt):

        # fmt is a string of tokens like 'YYYY-MM-DD'
        # we construct a new string by replacing each
        # token by its pattern:
        # 'YYYY-MM-DD' -> '(?P<YYYY>\d{4})-(?P<MM>\d{2})-(?P<DD>\d{2})'
        tokens = []
        offset = 0

        # Extract the bracketed expressions to be reinserted later.
        escaped_fmt = re.sub(self._ESCAPE_RE, "#", fmt)
        # Any number of S is the same as one.
        escaped_fmt = re.sub('S+', 'S', escaped_fmt)
        escaped_data = re.findall(self._ESCAPE_RE, fmt)

        fmt_pattern = escaped_fmt

        for m in self._FORMAT_RE.finditer(escaped_fmt):
            token = m.group(0)
            try:
                input_re = self._input_re_map[token]
            except KeyError:
                raise ParserError('Unrecognized token \'{0}\''.format(token))
            input_pattern = '(?P<{0}>{1})'.format(token, input_re.pattern)
            tokens.append(token)
            # a pattern doesn't have the same length as the token
            # it replaces! We keep the difference in the offset variable.
            # This works because the string is scanned left-to-right and matches
            # are returned in the order found by finditer.
            fmt_pattern = fmt_pattern[:m.start() + offset] + input_pattern + fmt_pattern[m.end() + offset:]
            offset += len(input_pattern) - (m.end() - m.start())

        final_fmt_pattern = ""
        a = fmt_pattern.split("#")
        b = escaped_data

        # Due to the way Python splits, 'a' will always be longer
        for i in range(len(a)):
            final_fmt_pattern += a[i]
            if i < len(b):
                final_fmt_pattern += b[i][1:-1]

        return tokens, re.compile(final_fmt_pattern, flags=re.IGNORECASE)

    def parse(self, string, fmt):

        if isinstance(fmt, list):
            return self._parse_multiformat(string, fmt)

        fmt_tokens, fmt_pattern_re = self._generate_pattern_re(fmt)

        match = fmt_pattern_re.search(string)
        if match is None:
            raise ParserError('Failed to match \'{0}\' when parsing \'{1}\''
                              .format(fmt_pattern_re.pattern, string))
        parts = {}
        for token in fmt_tokens:
            if token == 'Do':
                value = match.group('value')
            else:
                value = match.group(token)
            self._parse_token(token, value, parts)
        return self._build_datetime(parts)

    def _parse_token(self, token, value, parts):

        if token == 'YYYY':
            parts['year'] = int(value)
        elif token == 'YY':
            value = int(value)
            parts['year'] = 1900 + value if value > 68 else 2000 + value

        elif token in ['MMMM', 'MMM']:
            parts['month'] = self.locale.month_number(value.lower())

        elif token in ['MM', 'M']:
            parts['month'] = int(value)

        elif token in ['DD', 'D']:
            parts['day'] = int(value)

        elif token in ['Do']:
            parts['day'] = int(value)

        elif token.upper() in ['HH', 'H']:
            parts['hour'] = int(value)

        elif token in ['mm', 'm']:
            parts['minute'] = int(value)

        elif token in ['ss', 's']:
            parts['second'] = int(value)

        elif token == 'S':
            # We have the *most significant* digits of an arbitrary-precision integer.
            # We want the six most significant digits as an integer, rounded.
            # FIXME: add nanosecond support somehow?
            value = value.ljust(7, str('0'))

            # floating-point (IEEE-754) defaults to half-to-even rounding
            seventh_digit = int(value[6])
            if seventh_digit == 5:
                rounding = int(value[5]) % 2
            elif seventh_digit > 5:
                rounding = 1
            else:
                rounding = 0

            parts['microsecond'] = int(value[:6]) + rounding

        elif token == 'X':
            parts['timestamp'] = int(value)

        elif token in ['ZZZ', 'ZZ', 'Z']:
            parts['tzinfo'] = TzinfoParser.parse(value)

        elif token in ['a', 'A']:
            if value in (
                    self.locale.meridians['am'],
                    self.locale.meridians['AM']
            ):
                parts['am_pm'] = 'am'
            elif value in (
                    self.locale.meridians['pm'],
                    self.locale.meridians['PM']
            ):
                parts['am_pm'] = 'pm'

    @staticmethod
    def _build_datetime(parts):

        timestamp = parts.get('timestamp')

        if timestamp:
            tz_utc = tz.tzutc()
            return datetime.fromtimestamp(timestamp, tz=tz_utc)

        am_pm = parts.get('am_pm')
        hour = parts.get('hour', 0)

        if am_pm == 'pm' and hour < 12:
            hour += 12
        elif am_pm == 'am' and hour == 12:
            hour = 0

        return datetime(year=parts.get('year', 1), month=parts.get('month', 1),
            day=parts.get('day', 1), hour=hour, minute=parts.get('minute', 0),
            second=parts.get('second', 0), microsecond=parts.get('microsecond', 0),
            tzinfo=parts.get('tzinfo'))

    def _parse_multiformat(self, string, formats):

        _datetime = None

        for fmt in formats:
            try:
                _datetime = self.parse(string, fmt)
                break
            except ParserError:
                pass

        if _datetime is None:
            raise ParserError('Could not match input to any of {0} on \'{1}\''.format(formats, string))

        return _datetime

    @staticmethod
    def _map_lookup(input_map, key):

        try:
            return input_map[key]
        except KeyError:
            raise ParserError('Could not match "{0}" to {1}'.format(key, input_map))

    @staticmethod
    def _try_timestamp(string):

        try:
            return float(string)
        except:
            return None

    @staticmethod
    def _choice_re(choices, flags=0):
        return re.compile('({0})'.format('|'.join(choices)), flags=flags)

class TzinfoParser(object):

    _TZINFO_RE = re.compile('([+\-])?(\d\d):?(\d\d)?')

    @classmethod
    def parse(cls, string):

        tzinfo = None

        if string == 'local':
            tzinfo = tz.tzlocal()

        elif string in ['utc', 'UTC']:
            tzinfo = tz.tzutc()

        else:

            iso_match = cls._TZINFO_RE.match(string)

            if iso_match:
                sign, hours, minutes = iso_match.groups()
                if minutes is None:
                    minutes = 0
                seconds = int(hours) * 3600 + int(minutes) * 60

                if sign == '-':
                    seconds *= -1

                tzinfo = tz.tzoffset(None, seconds)

            else:
                tzinfo = tz.gettz(string)

        if tzinfo is None:
            raise ParserError('Could not parse timezone expression "{0}"'.format(string))

        return tzinfo
<commit_msg>fixed coverage to ignore Py2 only imports<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals

from datetime import datetime
from dateutil import tz
import re

try:
    from functools import lru_cache
except ImportError:  # pragma: no cover
    from backports.functools_lru_cache import lru_cache  # pragma: no cover

from arrow import locales

class ParserError(RuntimeError):
    pass

class DateTimeParser(object):

    _FORMAT_RE = re.compile('(YYY?Y?|MM?M?M?|Do|DD?D?D?|d?d?d?d|HH?|hh?|mm?|ss?|S+|ZZ?Z?|a|A|X)')
    _ESCAPE_RE = re.compile('\[[^\[\]]*\]')

    _ONE_OR_MORE_DIGIT_RE = re.compile('\d+')
    _ONE_OR_TWO_DIGIT_RE = re.compile('\d{1,2}')
    _FOUR_DIGIT_RE = re.compile('\d{4}')
    _TWO_DIGIT_RE = re.compile('\d{2}')
    _TZ_RE = re.compile('[+\-]?\d{2}:?(\d{2})?')
    _TZ_NAME_RE = re.compile('\w[\w+\-/]+')

    _BASE_INPUT_RE_MAP = {
        'YYYY': _FOUR_DIGIT_RE,
        'YY': _TWO_DIGIT_RE,
        'MM': _TWO_DIGIT_RE,
        'M': _ONE_OR_TWO_DIGIT_RE,
        'DD': _TWO_DIGIT_RE,
        'D': _ONE_OR_TWO_DIGIT_RE,
        'HH': _TWO_DIGIT_RE,
        'H': _ONE_OR_TWO_DIGIT_RE,
        'hh': _TWO_DIGIT_RE,
        'h': _ONE_OR_TWO_DIGIT_RE,
        'mm': _TWO_DIGIT_RE,
        'm': _ONE_OR_TWO_DIGIT_RE,
        'ss': _TWO_DIGIT_RE,
        's': _ONE_OR_TWO_DIGIT_RE,
        'X': re.compile('\d+'),
        'ZZZ': _TZ_NAME_RE,
        'ZZ': _TZ_RE,
        'Z': _TZ_RE,
        'S': _ONE_OR_MORE_DIGIT_RE,
    }

    MARKERS = ['YYYY', 'MM', 'DD']
    SEPARATORS = ['-', '/', '.']

    def __init__(self, locale='en_us', cache_size=0):

        self.locale = locales.get_locale(locale)
        self._input_re_map = self._BASE_INPUT_RE_MAP.copy()
        self._input_re_map.update({
            'MMMM': self._choice_re(self.locale.month_names[1:], re.IGNORECASE),
            'MMM': self._choice_re(self.locale.month_abbreviations[1:],
                                   re.IGNORECASE),
            'Do': re.compile(self.locale.ordinal_day_re),
            'dddd': self._choice_re(self.locale.day_names[1:], re.IGNORECASE),
            'ddd': self._choice_re(self.locale.day_abbreviations[1:],
                                   re.IGNORECASE),
            'd': re.compile(r"[1-7]"),
            'a': self._choice_re(
                (self.locale.meridians['am'], self.locale.meridians['pm'])
            ),
            # note: 'A' token accepts both 'am/pm' and 'AM/PM' formats to
            # ensure backwards compatibility of this token
            'A': self._choice_re(self.locale.meridians.values())
        })
        if cache_size > 0:
            self._generate_pattern_re =\
                lru_cache(maxsize=cache_size)(self._generate_pattern_re)

    def parse_iso(self, string):

        has_time = 'T' in string or ' ' in string.strip()
        space_divider = ' ' in string.strip()

        if has_time:
            if space_divider:
                date_string, time_string = string.split(' ', 1)
            else:
                date_string, time_string = string.split('T', 1)
            time_parts = re.split('[+-]', time_string, 1)
            has_tz = len(time_parts) > 1
            has_seconds = time_parts[0].count(':') > 1
            has_subseconds = re.search('[.,]', time_parts[0])

            if has_subseconds:
                formats = ['YYYY-MM-DDTHH:mm:ss%sS' % has_subseconds.group()]
            elif has_seconds:
                formats = ['YYYY-MM-DDTHH:mm:ss']
            else:
                formats = ['YYYY-MM-DDTHH:mm']
        else:
            has_tz = False
            # generate required formats: YYYY-MM-DD, YYYY-MM-DD, YYYY
            # using various separators: -, /, .
            l = len(self.MARKERS)
            formats = [separator.join(self.MARKERS[:l-i])
                       for i in range(l)
                       for separator in self.SEPARATORS]

        if has_time and has_tz:
            formats = [f + 'Z' for f in formats]

        if space_divider:
            formats = [item.replace('T', ' ', 1) for item in formats]

        return self._parse_multiformat(string, formats)

    def _generate_pattern_re(self, fmt):

        # fmt is a string of tokens like 'YYYY-MM-DD'
        # we construct a new string by replacing each
        # token by its pattern:
        # 'YYYY-MM-DD' -> '(?P<YYYY>\d{4})-(?P<MM>\d{2})-(?P<DD>\d{2})'
        tokens = []
        offset = 0

        # Extract the bracketed expressions to be reinserted later.
        escaped_fmt = re.sub(self._ESCAPE_RE, "#", fmt)
        # Any number of S is the same as one.
        escaped_fmt = re.sub('S+', 'S', escaped_fmt)
        escaped_data = re.findall(self._ESCAPE_RE, fmt)

        fmt_pattern = escaped_fmt

        for m in self._FORMAT_RE.finditer(escaped_fmt):
            token = m.group(0)
            try:
                input_re = self._input_re_map[token]
            except KeyError:
                raise ParserError('Unrecognized token \'{0}\''.format(token))
            input_pattern = '(?P<{0}>{1})'.format(token, input_re.pattern)
            tokens.append(token)
            # a pattern doesn't have the same length as the token
            # it replaces! We keep the difference in the offset variable.
            # This works because the string is scanned left-to-right and matches
            # are returned in the order found by finditer.
            fmt_pattern = fmt_pattern[:m.start() + offset] + input_pattern + fmt_pattern[m.end() + offset:]
            offset += len(input_pattern) - (m.end() - m.start())

        final_fmt_pattern = ""
        a = fmt_pattern.split("#")
        b = escaped_data

        # Due to the way Python splits, 'a' will always be longer
        for i in range(len(a)):
            final_fmt_pattern += a[i]
            if i < len(b):
                final_fmt_pattern += b[i][1:-1]

        return tokens, re.compile(final_fmt_pattern, flags=re.IGNORECASE)

    def parse(self, string, fmt):

        if isinstance(fmt, list):
            return self._parse_multiformat(string, fmt)

        fmt_tokens, fmt_pattern_re = self._generate_pattern_re(fmt)

        match = fmt_pattern_re.search(string)
        if match is None:
            raise ParserError('Failed to match \'{0}\' when parsing \'{1}\''
                              .format(fmt_pattern_re.pattern, string))
        parts = {}
        for token in fmt_tokens:
            if token == 'Do':
                value = match.group('value')
            else:
                value = match.group(token)
            self._parse_token(token, value, parts)
        return self._build_datetime(parts)

    def _parse_token(self, token, value, parts):

        if token == 'YYYY':
            parts['year'] = int(value)
        elif token == 'YY':
            value = int(value)
            parts['year'] = 1900 + value if value > 68 else 2000 + value

        elif token in ['MMMM', 'MMM']:
            parts['month'] = self.locale.month_number(value.lower())

        elif token in ['MM', 'M']:
            parts['month'] = int(value)

        elif token in ['DD', 'D']:
            parts['day'] = int(value)

        elif token in ['Do']:
            parts['day'] = int(value)

        elif token.upper() in ['HH', 'H']:
            parts['hour'] = int(value)

        elif token in ['mm', 'm']:
            parts['minute'] = int(value)

        elif token in ['ss', 's']:
            parts['second'] = int(value)

        elif token == 'S':
            # We have the *most significant* digits of an arbitrary-precision integer.
            # We want the six most significant digits as an integer, rounded.
            # FIXME: add nanosecond support somehow?
            value = value.ljust(7, str('0'))

            # floating-point (IEEE-754) defaults to half-to-even rounding
            seventh_digit = int(value[6])
            if seventh_digit == 5:
                rounding = int(value[5]) % 2
            elif seventh_digit > 5:
                rounding = 1
            else:
                rounding = 0

            parts['microsecond'] = int(value[:6]) + rounding

        elif token == 'X':
            parts['timestamp'] = int(value)

        elif token in ['ZZZ', 'ZZ', 'Z']:
            parts['tzinfo'] = TzinfoParser.parse(value)

        elif token in ['a', 'A']:
            if value in (
                    self.locale.meridians['am'],
                    self.locale.meridians['AM']
            ):
                parts['am_pm'] = 'am'
            elif value in (
                    self.locale.meridians['pm'],
                    self.locale.meridians['PM']
            ):
                parts['am_pm'] = 'pm'

    @staticmethod
    def _build_datetime(parts):

        timestamp = parts.get('timestamp')

        if timestamp:
            tz_utc = tz.tzutc()
            return datetime.fromtimestamp(timestamp, tz=tz_utc)

        am_pm = parts.get('am_pm')
        hour = parts.get('hour', 0)

        if am_pm == 'pm' and hour < 12:
            hour += 12
        elif am_pm == 'am' and hour == 12:
            hour = 0

        return datetime(year=parts.get('year', 1), month=parts.get('month', 1),
            day=parts.get('day', 1), hour=hour, minute=parts.get('minute', 0),
            second=parts.get('second', 0), microsecond=parts.get('microsecond', 0),
            tzinfo=parts.get('tzinfo'))

    def _parse_multiformat(self, string, formats):

        _datetime = None

        for fmt in formats:
            try:
                _datetime = self.parse(string, fmt)
                break
            except ParserError:
                pass

        if _datetime is None:
            raise ParserError('Could not match input to any of {0} on \'{1}\''.format(formats, string))

        return _datetime

    @staticmethod
    def _map_lookup(input_map, key):

        try:
            return input_map[key]
        except KeyError:
            raise ParserError('Could not match "{0}" to {1}'.format(key, input_map))

    @staticmethod
    def _try_timestamp(string):

        try:
            return float(string)
        except:
            return None

    @staticmethod
    def _choice_re(choices, flags=0):
        return re.compile('({0})'.format('|'.join(choices)), flags=flags)

class TzinfoParser(object):

    _TZINFO_RE = re.compile('([+\-])?(\d\d):?(\d\d)?')

    @classmethod
    def parse(cls, string):

        tzinfo = None

        if string == 'local':
            tzinfo = tz.tzlocal()

        elif string in ['utc', 'UTC']:
            tzinfo = tz.tzutc()

        else:

            iso_match = cls._TZINFO_RE.match(string)

            if iso_match:
                sign, hours, minutes = iso_match.groups()
                if minutes is None:
                    minutes = 0
                seconds = int(hours) * 3600 + int(minutes) * 60

                if sign == '-':
                    seconds *= -1

                tzinfo = tz.tzoffset(None, seconds)

            else:
                tzinfo = tz.gettz(string)

        if tzinfo is None:
            raise ParserError('Could not parse timezone expression "{0}"'.format(string))

        return tzinfo
<|endoftext|>


<commit_before>{
  "targets": [
    {
      "target_name": "ghostscript4js",
      "sources": ["src/ghostscript4js.cc"],
      "include_dirs": [
        "<!(node -e \"require('nan')\")"
      ],
      "libraries": [
        "/usr/lib/x86_64-linux-gnu/libgs.so"]
    }
  ]
}
<commit_msg>Updated settings binding.gyp for linux and mac<commit_after>{
  "targets": [
    {
      "target_name": "ghostscript4js",
      "sources": ["src/ghostscript4js.cc"],
      "include_dirs": [
        "<!(node -e \"require('nan')\")"
      ],
      "conditions": [
        ['OS=="linux"', {
          'variables': {
            "GS4JS_LIB%": "<!(echo $GS4JS_ENV_LIB)",
            "conditions": [
              ['"<!(echo $GS4JS_ENV_LIB)" == ""', {
                "GS4JS_LIB%": "/usr/lib/x86_64-linux-gnu/libgs.so"
              }]
            ],
          },
          "libraries": ["<(GS4JS_LIB)"]
        }],
        ['OS=="win"', {

        }],
        ['OS=="mac"', {
          'variables': {
            "GS4JS_LIB%": "<!(echo $GS4JS_ENV_LIB)",
            "conditions": [
              ['"<!(echo $GS4JS_ENV_LIB)" == ""', {
                "GS4JS_LIB%": "/usr/local/lib/libgs.dylib"
              }]
            ]
          },
          "libraries": ["<(GS4JS_LIB)"]
        }]
      ]
    }
  ]
}
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import urllib
import config as c
from pymongo import MongoClient

client = MongoClient()
db = client.articles.entries

def generate_filename(entry, directory = ''):
  authors = [a.split()[-1] for a in entry['authors']]
  authors = authors[0]+'_et.al' if len(authors) > 1 else authors[0]
  title = entry['title'].replace(' ', '_')
  return ''.join([directory, authors,'-',title , '.pdf'])

def dl_pdf_from_arxiv(url):
  html_doc = urllib.urlopen(url).read()
  s = BeautifulSoup(html_doc)
  entries = [{
    'pdf' : e.findAll('link',attrs={'type': 'application/pdf'})[0]['href'],
    'url' : e.findAll('link',attrs={'type': 'text/html'})[0]['href'],
    'authors': [str(a.next.next.next) for a in e.findAll('author')],
    'title': str(e.title.next),
    'id': str.split(str(e.id.next),'/')[-1]
            } for e in s.findAll('entry')]
  entries = filter(lambda e: db.find_one({'id': e['id']}) == None, entries)
  map(lambda e: urllib.urlretrieve(e['pdf'], generate_filename(e, directory = c.ARTICLE_DIR)), entries)
  if entries:
    db.insert(entries)

<commit_msg>do more with less steps<commit_after>from bs4 import BeautifulSoup
import urllib
import config as c
from pymongo import MongoClient

client = MongoClient()
db = client.articles.entries

def generate_filename(entry, directory = ''):
  authors = [a.split()[-1] for a in entry['authors']]
  authors = authors[0]+'_et.al' if len(authors) > 1 else authors[0]
  title = entry['title'].replace(' ', '_')
  return ''.join([directory, authors,'-',title , '.pdf'])

def dl_pdf_from_arxiv(url):
  html_doc = urllib.urlopen(url).read()
  s = BeautifulSoup(html_doc)
  entries = [{
    'pdf' : e.findAll('link',attrs={'type': 'application/pdf'})[0]['href'],
    'url' : e.findAll('link',attrs={'type': 'text/html'})[0]['href'],
    'authors': [str(a.next.next.next) for a in e.findAll('author')],
    'title': str(e.title.next),
    'id': str.split(str(e.id.next),'/')[-1],
            } for e in s.findAll('entry')]
  entries = filter(lambda e: db.find_one({'id': e['id']}) == None,
                   [entry['path'] = generate_filename(entry, directory = c.ARTICLE_DIR for entry in entries)
  map(lambda e: urllib.urlretrieve(e['pdf'], generate_filename(e, directory = c.ARTICLE_DIR)), entries)
  if entries:
    db.insert(entries)

<|endoftext|>


<commit_before>"""Client for Google's BrowserChannel protocol.

BrowserChannel allows simulating a bidirectional socket in a web browser using
long-polling requests. It is used by the Hangouts web client to receive state
updates from the server. The "forward channel" sends "maps" (dictionaries) to
the server. The "backwards channel" receives "arrays" (lists) from the server.

Google provides a JavaScript BrowserChannel client as part of closure-library:
http://google.github.io/closure-library/api/class_goog_net_BrowserChannel.html

Source code is available here:
https://github.com/google/closure-library/blob/master/closure/goog/net/browserchannel.js

Unofficial protocol documentation is available here:
https://web.archive.org/web/20121226064550/http://code.google.com/p/libevent-browserchannel-server/wiki/BrowserChannelProtocol
"""

import asyncio
import codecs
import json
import logging
import re

import aiohttp
import async_timeout

from hangups import event, exceptions

logger = logging.getLogger(__name__)
Utf8IncrementalDecoder = codecs.getincrementaldecoder('utf-8')
LEN_REGEX = re.compile(r'([0-9]+)\n', re.MULTILINE)
CHANNEL_URL = 'https://0.client-channel.google.com/client-channel/channel/bind'
# Long-polling requests send heartbeats every 15-30 seconds, so if we miss two
# in a row, consider the connection dead.
PUSH_TIMEOUT = 60
MAX_READ_BYTES = 1024 * 1024

class UnknownSIDError(exceptions.HangupsError):

    """hangups channel session expired."""

    pass

def _best_effort_decode(data_bytes):
    """Decode as much of data_bytes as possible as UTF-8."""
    decoder = Utf8IncrementalDecoder()
    return decoder.decode(data_bytes)

class ChunkParser(object):
    """Parse data from the backward channel into chunks.

    Responses from the backward channel consist of a sequence of chunks which
    are streamed to the client. Each chunk is prefixed with its length,
    followed by a newline. The length allows the client to identify when the
    entire chunk has been received.
    """

    def __init__(self):
        # Buffer for bytes containing utf-8 text:
        self._buf = b''

    def get_chunks(self, new_data_bytes):
        """Yield chunks generated from received data.

        The buffer may not be decodable as UTF-8 if there's a split multi-byte
        character at the end. To handle this, do a "best effort" decode of the
        buffer to decode as much of it as possible.

        The length is actually the length of the string as reported by
        JavaScript. JavaScript's string length function returns the number of
        code units in the string, represented in UTF-16. We can emulate this by
        encoding everything in UTF-16 and multiplying the reported length by 2.

        Note that when encoding a string in UTF-16, Python will prepend a
        byte-order character, so we need to remove the first two bytes.
        """
        self._buf += new_data_bytes

        while True:

            buf_decoded = _best_effort_decode(self._buf)
            buf_utf16 = buf_decoded.encode('utf-16')[2:]

            length_str_match = LEN_REGEX.match(buf_decoded)
            if length_str_match is None:
                break
            else:
                length_str = length_str_match.group(1)
                # Both lengths are in number of bytes in UTF-16 encoding.
                # The length of the submission:
                length = int(length_str) * 2
                # The length of the submission length and newline:
                length_length = len((length_str + '\n').encode('utf-16')[2:])
                if len(buf_utf16) - length_length < length:
                    break

                submission = buf_utf16[length_length:length_length + length]
                yield submission.decode('utf-16')
                # Drop the length and the submission itself from the beginning
                # of the buffer.
                drop_length = (len((length_str + '\n').encode()) +
                               len(submission.decode('utf-16').encode()))
                self._buf = self._buf[drop_length:]

def _parse_sid_response(res):
    """Parse response format for request for new channel SID.

    Example format (after parsing JS):
    [   [0,["c","SID_HERE","",8]],
        [1,[{"gsid":"GSESSIONID_HERE"}]]]

    Returns (SID, gsessionid) tuple.
    """
    res = json.loads(list(ChunkParser().get_chunks(res))[0])
    sid = res[0][1][1]
    gsessionid = res[1][1][0]['gsid']
    return (sid, gsessionid)

class Channel(object):
    """BrowserChannel client."""

    ##########################################################################
    # Public methods
    ##########################################################################

    def __init__(self, session, max_retries, retry_backoff_base):
        """Create a new channel.

        Args:
            session (http_utils.Session): Request session.
            max_retries (int): Number of retries for long-polling request.
            retry_backoff_base (int): The base term for the long-polling
                exponential backoff.
        """

        # Event fired when channel connects with arguments ():
        self.on_connect = event.Event('Channel.on_connect')
        # Event fired when channel reconnects with arguments ():
        self.on_reconnect = event.Event('Channel.on_reconnect')
        # Event fired when channel disconnects with arguments ():
        self.on_disconnect = event.Event('Channel.on_disconnect')
        # Event fired when an array is received with arguments (array):
        self.on_receive_array = event.Event('Channel.on_receive_array')

        self._max_retries = max_retries
        self._retry_backoff_base = retry_backoff_base

        # True if the channel is currently connected:
        self._is_connected = False
        # True if the on_connect event has been called at least once:
        self._on_connect_called = False
        # Parser for assembling messages:
        self._chunk_parser = None
        # Session for HTTP requests:
        self._session = session

        # Discovered parameters:
        self._sid_param = None
        self._gsessionid_param = None

    @property
    def is_connected(self):
        """Whether the channel is currently connected."""
        return self._is_connected

    async def listen(self):
        """Listen for messages on the backwards channel.

        This method only returns when the connection has been closed due to an
        error.
        """
        retries = 0  # Number of retries attempted so far
        need_new_sid = True  # whether a new SID is needed

        while retries <= self._max_retries:
            # After the first failed retry, back off exponentially longer after
            # each attempt.
            if retries > 0:
                backoff_seconds = self._retry_backoff_base ** retries
                logger.info('Backing off for %s seconds', backoff_seconds)
                await asyncio.sleep(backoff_seconds)

            # Request a new SID if we don't have one yet, or the previous one
            # became invalid.
            if need_new_sid:
                await self._fetch_channel_sid()
                need_new_sid = False
            # Clear any previous push data, since if there was an error it
            # could contain garbage.
            self._chunk_parser = ChunkParser()
            try:
                await self._longpoll_request()
            except (UnknownSIDError, exceptions.NetworkError) as e:
                logger.warning('Long-polling request failed: {}'.format(e))
                retries += 1
                logger.warning('retry attempt count is now {}'.format(retries))
                if self._is_connected:
                    self._is_connected = False
                    await self.on_disconnect.fire()
                if isinstance(e, UnknownSIDError):
                    need_new_sid = True
            else:
                # The connection closed successfully, so reset the number of
                # retries.
                retries = 0

            # If the request ended with an error, the client must account for
            # messages being dropped during this time.

        logger.error('Ran out of retries for long-polling request')

    async def send_maps(self, map_list):
        """Sends a request to the server containing maps (dicts)."""
        params = {
            'VER': 8,  # channel protocol version
            'RID': 81188,  # request identifier
            'ctype': 'hangouts',  # client type
        }
        if self._gsessionid_param is not None:
            params['gsessionid'] = self._gsessionid_param
        if self._sid_param is not None:
            params['SID'] = self._sid_param
        data_dict = dict(count=len(map_list), ofs=0)
        for map_num, map_ in enumerate(map_list):
            for map_key, map_val in map_.items():
                data_dict['req{}_{}'.format(map_num, map_key)] = map_val
        res = await self._session.fetch(
            'post', CHANNEL_URL, params=params, data=data_dict
        )
        return res

    ##########################################################################
    # Private methods
    ##########################################################################

    async def _fetch_channel_sid(self):
        """Creates a new channel for receiving push data.

        Sending an empty forward channel request will create a new channel on
        the server.

        There's a separate API to get the gsessionid alone that Hangouts for
        Chrome uses, but if we don't send a gsessionid with this request, it
        will return a gsessionid as well as the SID.

        Raises hangups.NetworkError if the channel can not be created.
        """
        logger.info('Requesting new gsessionid and SID...')
        # Set SID and gsessionid to None so they aren't sent in by send_maps.
        self._sid_param = None
        self._gsessionid_param = None
        res = await self.send_maps([])
        self._sid_param, self._gsessionid_param = _parse_sid_response(res.body)
        logger.info('New SID: {}'.format(self._sid_param))
        logger.info('New gsessionid: {}'.format(self._gsessionid_param))

    async def _longpoll_request(self):
        """Open a long-polling request and receive arrays.

        This method uses keep-alive to make re-opening the request faster, but
        the remote server will set the "Connection: close" header once an hour.

        Raises hangups.NetworkError or UnknownSIDError.
        """
        params = {
            'VER': 8,  # channel protocol version
            'gsessionid': self._gsessionid_param,
            'RID': 'rpc',  # request identifier
            't': 1,  # trial
            'SID': self._sid_param,  # session ID
            'CI': 0,  # 0 if streaming/chunked requests should be used
            'ctype': 'hangouts',  # client type
            'TYPE': 'xmlhttp',  # type of request
        }
        logger.info('Opening new long-polling request')
        try:
            async with self._session.fetch_raw('GET', CHANNEL_URL,
                                               params=params) as res:

                if res.status != 200:
                    if res.status == 400 and res.reason == 'Unknown SID':
                        raise UnknownSIDError('SID became invalid')
                    raise exceptions.NetworkError(
                        'Request return unexpected status: {}: {}'.format(
                            res.status, res.reason))

                while True:
                    async with async_timeout.timeout(PUSH_TIMEOUT):
                        chunk = await res.content.read(MAX_READ_BYTES)
                    if not chunk:
                        break

                    await self._on_push_data(chunk)

        except asyncio.TimeoutError:
            raise exceptions.NetworkError('Request timed out')
        except aiohttp.ServerDisconnectedError as err:
            raise exceptions.NetworkError(
                'Server disconnected error: %s' % err)
        except aiohttp.ClientError as err:
            raise exceptions.NetworkError('Request connection error: %s' % err)

    async def _on_push_data(self, data_bytes):
        """Parse push data and trigger events."""
        logger.debug('Received chunk:\n{}'.format(data_bytes))
        for chunk in self._chunk_parser.get_chunks(data_bytes):

            # Consider the channel connected once the first chunk is received.
            if not self._is_connected:
                if self._on_connect_called:
                    self._is_connected = True
                    await self.on_reconnect.fire()
                else:
                    self._on_connect_called = True
                    self._is_connected = True
                    await self.on_connect.fire()

            # chunk contains a container array
            container_array = json.loads(chunk)
            # container array is an array of inner arrays
            for inner_array in container_array:
                # inner_array always contains 2 elements, the array_id and the
                # data_array.
                array_id, data_array = inner_array
                logger.debug('Chunk contains data array with id %r:\n%r',
                             array_id, data_array)
                await self.on_receive_array.fire(data_array)
<commit_msg>[channel] simplify error handling of long-polling requests<commit_after>"""Client for Google's BrowserChannel protocol.

BrowserChannel allows simulating a bidirectional socket in a web browser using
long-polling requests. It is used by the Hangouts web client to receive state
updates from the server. The "forward channel" sends "maps" (dictionaries) to
the server. The "backwards channel" receives "arrays" (lists) from the server.

Google provides a JavaScript BrowserChannel client as part of closure-library:
http://google.github.io/closure-library/api/class_goog_net_BrowserChannel.html

Source code is available here:
https://github.com/google/closure-library/blob/master/closure/goog/net/browserchannel.js

Unofficial protocol documentation is available here:
https://web.archive.org/web/20121226064550/http://code.google.com/p/libevent-browserchannel-server/wiki/BrowserChannelProtocol
"""

import asyncio
import codecs
import json
import logging
import re

import aiohttp
import async_timeout

from hangups import event, exceptions

logger = logging.getLogger(__name__)
Utf8IncrementalDecoder = codecs.getincrementaldecoder('utf-8')
LEN_REGEX = re.compile(r'([0-9]+)\n', re.MULTILINE)
CHANNEL_URL = 'https://0.client-channel.google.com/client-channel/channel/bind'
# Long-polling requests send heartbeats every 15-30 seconds, so if we miss two
# in a row, consider the connection dead.
PUSH_TIMEOUT = 60
MAX_READ_BYTES = 1024 * 1024

class UnknownSIDError(exceptions.HangupsError):

    """hangups channel session expired."""

    pass

def _best_effort_decode(data_bytes):
    """Decode as much of data_bytes as possible as UTF-8."""
    decoder = Utf8IncrementalDecoder()
    return decoder.decode(data_bytes)

class ChunkParser(object):
    """Parse data from the backward channel into chunks.

    Responses from the backward channel consist of a sequence of chunks which
    are streamed to the client. Each chunk is prefixed with its length,
    followed by a newline. The length allows the client to identify when the
    entire chunk has been received.
    """

    def __init__(self):
        # Buffer for bytes containing utf-8 text:
        self._buf = b''

    def get_chunks(self, new_data_bytes):
        """Yield chunks generated from received data.

        The buffer may not be decodable as UTF-8 if there's a split multi-byte
        character at the end. To handle this, do a "best effort" decode of the
        buffer to decode as much of it as possible.

        The length is actually the length of the string as reported by
        JavaScript. JavaScript's string length function returns the number of
        code units in the string, represented in UTF-16. We can emulate this by
        encoding everything in UTF-16 and multiplying the reported length by 2.

        Note that when encoding a string in UTF-16, Python will prepend a
        byte-order character, so we need to remove the first two bytes.
        """
        self._buf += new_data_bytes

        while True:

            buf_decoded = _best_effort_decode(self._buf)
            buf_utf16 = buf_decoded.encode('utf-16')[2:]

            length_str_match = LEN_REGEX.match(buf_decoded)
            if length_str_match is None:
                break
            else:
                length_str = length_str_match.group(1)
                # Both lengths are in number of bytes in UTF-16 encoding.
                # The length of the submission:
                length = int(length_str) * 2
                # The length of the submission length and newline:
                length_length = len((length_str + '\n').encode('utf-16')[2:])
                if len(buf_utf16) - length_length < length:
                    break

                submission = buf_utf16[length_length:length_length + length]
                yield submission.decode('utf-16')
                # Drop the length and the submission itself from the beginning
                # of the buffer.
                drop_length = (len((length_str + '\n').encode()) +
                               len(submission.decode('utf-16').encode()))
                self._buf = self._buf[drop_length:]

def _parse_sid_response(res):
    """Parse response format for request for new channel SID.

    Example format (after parsing JS):
    [   [0,["c","SID_HERE","",8]],
        [1,[{"gsid":"GSESSIONID_HERE"}]]]

    Returns (SID, gsessionid) tuple.
    """
    res = json.loads(list(ChunkParser().get_chunks(res))[0])
    sid = res[0][1][1]
    gsessionid = res[1][1][0]['gsid']
    return (sid, gsessionid)

class Channel(object):
    """BrowserChannel client."""

    ##########################################################################
    # Public methods
    ##########################################################################

    def __init__(self, session, max_retries, retry_backoff_base):
        """Create a new channel.

        Args:
            session (http_utils.Session): Request session.
            max_retries (int): Number of retries for long-polling request.
            retry_backoff_base (int): The base term for the long-polling
                exponential backoff.
        """

        # Event fired when channel connects with arguments ():
        self.on_connect = event.Event('Channel.on_connect')
        # Event fired when channel reconnects with arguments ():
        self.on_reconnect = event.Event('Channel.on_reconnect')
        # Event fired when channel disconnects with arguments ():
        self.on_disconnect = event.Event('Channel.on_disconnect')
        # Event fired when an array is received with arguments (array):
        self.on_receive_array = event.Event('Channel.on_receive_array')

        self._max_retries = max_retries
        self._retry_backoff_base = retry_backoff_base

        # True if the channel is currently connected:
        self._is_connected = False
        # True if the on_connect event has been called at least once:
        self._on_connect_called = False
        # Parser for assembling messages:
        self._chunk_parser = None
        # Session for HTTP requests:
        self._session = session

        # Discovered parameters:
        self._sid_param = None
        self._gsessionid_param = None

    @property
    def is_connected(self):
        """Whether the channel is currently connected."""
        return self._is_connected

    async def listen(self):
        """Listen for messages on the backwards channel.

        This method only returns when the connection has been closed due to an
        error.
        """
        retries = 0  # Number of retries attempted so far
        need_new_sid = True  # whether a new SID is needed

        while retries <= self._max_retries:
            # After the first failed retry, back off exponentially longer after
            # each attempt.
            if retries > 0:
                backoff_seconds = self._retry_backoff_base ** retries
                logger.info('Backing off for %s seconds', backoff_seconds)
                await asyncio.sleep(backoff_seconds)

            # Request a new SID if we don't have one yet, or the previous one
            # became invalid.
            if need_new_sid:
                await self._fetch_channel_sid()
                need_new_sid = False
            # Clear any previous push data, since if there was an error it
            # could contain garbage.
            self._chunk_parser = ChunkParser()
            try:
                await self._longpoll_request()
            except UnknownSIDError as err:
                logger.warning('Long-polling interrupted: %s', err)
                need_new_sid = True
            except exceptions.NetworkError as err:
                logger.warning('Long-polling request failed: %s', err)
            else:
                # The connection closed successfully, so reset the number of
                # retries.
                retries = 0
                continue

            retries += 1
            logger.info('retry attempt count is now %s', retries)
            if self._is_connected:
                self._is_connected = False
                await self.on_disconnect.fire()

            # If the request ended with an error, the client must account for
            # messages being dropped during this time.

        logger.error('Ran out of retries for long-polling request')

    async def send_maps(self, map_list):
        """Sends a request to the server containing maps (dicts)."""
        params = {
            'VER': 8,  # channel protocol version
            'RID': 81188,  # request identifier
            'ctype': 'hangouts',  # client type
        }
        if self._gsessionid_param is not None:
            params['gsessionid'] = self._gsessionid_param
        if self._sid_param is not None:
            params['SID'] = self._sid_param
        data_dict = dict(count=len(map_list), ofs=0)
        for map_num, map_ in enumerate(map_list):
            for map_key, map_val in map_.items():
                data_dict['req{}_{}'.format(map_num, map_key)] = map_val
        res = await self._session.fetch(
            'post', CHANNEL_URL, params=params, data=data_dict
        )
        return res

    ##########################################################################
    # Private methods
    ##########################################################################

    async def _fetch_channel_sid(self):
        """Creates a new channel for receiving push data.

        Sending an empty forward channel request will create a new channel on
        the server.

        There's a separate API to get the gsessionid alone that Hangouts for
        Chrome uses, but if we don't send a gsessionid with this request, it
        will return a gsessionid as well as the SID.

        Raises hangups.NetworkError if the channel can not be created.
        """
        logger.info('Requesting new gsessionid and SID...')
        # Set SID and gsessionid to None so they aren't sent in by send_maps.
        self._sid_param = None
        self._gsessionid_param = None
        res = await self.send_maps([])
        self._sid_param, self._gsessionid_param = _parse_sid_response(res.body)
        logger.info('New SID: {}'.format(self._sid_param))
        logger.info('New gsessionid: {}'.format(self._gsessionid_param))

    async def _longpoll_request(self):
        """Open a long-polling request and receive arrays.

        This method uses keep-alive to make re-opening the request faster, but
        the remote server will set the "Connection: close" header once an hour.

        Raises hangups.NetworkError or UnknownSIDError.
        """
        params = {
            'VER': 8,  # channel protocol version
            'gsessionid': self._gsessionid_param,
            'RID': 'rpc',  # request identifier
            't': 1,  # trial
            'SID': self._sid_param,  # session ID
            'CI': 0,  # 0 if streaming/chunked requests should be used
            'ctype': 'hangouts',  # client type
            'TYPE': 'xmlhttp',  # type of request
        }
        logger.info('Opening new long-polling request')
        try:
            async with self._session.fetch_raw('GET', CHANNEL_URL,
                                               params=params) as res:

                if res.status != 200:
                    if res.status == 400 and res.reason == 'Unknown SID':
                        raise UnknownSIDError('SID became invalid')
                    raise exceptions.NetworkError(
                        'Request return unexpected status: {}: {}'.format(
                            res.status, res.reason))

                while True:
                    async with async_timeout.timeout(PUSH_TIMEOUT):
                        chunk = await res.content.read(MAX_READ_BYTES)
                    if not chunk:
                        break

                    await self._on_push_data(chunk)

        except asyncio.TimeoutError:
            raise exceptions.NetworkError('Request timed out')
        except aiohttp.ServerDisconnectedError as err:
            raise exceptions.NetworkError(
                'Server disconnected error: %s' % err)
        except aiohttp.ClientError as err:
            raise exceptions.NetworkError('Request connection error: %s' % err)

    async def _on_push_data(self, data_bytes):
        """Parse push data and trigger events."""
        logger.debug('Received chunk:\n{}'.format(data_bytes))
        for chunk in self._chunk_parser.get_chunks(data_bytes):

            # Consider the channel connected once the first chunk is received.
            if not self._is_connected:
                if self._on_connect_called:
                    self._is_connected = True
                    await self.on_reconnect.fire()
                else:
                    self._on_connect_called = True
                    self._is_connected = True
                    await self.on_connect.fire()

            # chunk contains a container array
            container_array = json.loads(chunk)
            # container array is an array of inner arrays
            for inner_array in container_array:
                # inner_array always contains 2 elements, the array_id and the
                # data_array.
                array_id, data_array = inner_array
                logger.debug('Chunk contains data array with id %r:\n%r',
                             array_id, data_array)
                await self.on_receive_array.fire(data_array)
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.urlresolvers import reverse
from django.template import Template, Context
from django.test import TestCase
import unittest

class TestSharedContent(TestCase):

    def setUp(self):
        self.slug = 'admin'

    @unittest.skip("Makes multiple generous assumptions about project behaviour")
    def test_template_media(self):
        """
        Ensures base template has required media files.
        """
        response = self.client.get(reverse('home'))
        static_url = settings.STATIC_URL
        favicon_url = '<link rel="shortcut icon" href="{}img/favicon.png">'.format(static_url)
        touch_icon = '<link rel="apple-touch-icon" href="{}img/touch-icon.png">'.format(STATIC_URL)
        self.assertEqual(response.status_code, 200)
        self.assertTrue(favicon_url in response.content)
        self.assertTrue(touch_icon in response.content)

    @unittest.skip("Makes multiple generous assumptions about project behaviour")
    def test_shared_context_processor(self):
        """
        Test results of shared context processor are in template
        """
        response = self.client.get(reverse('home'))
        self.assertEqual(response.status_code, 200)
        self.assertTrue('site' in response.context)
        self.assertTrue('now' in response.context)
        self.assertTrue('year' in response.context)
        self.assertTrue('ga_code' in response.context)
        self.assertTrue('project_name' in response.context)
        self.assertTrue('current_path' in response.context)
        self.assertTrue('last_seen' in response.context)
        self.assertTrue('last_seen_fuzzy' in response.context)
        self.assertTrue('theme' in response.context)
        self.assertTrue('authenticated_request' in response.context)

class TemplateTagsTests(TestCase):
    def setUp(self):
        self.test_list = ['apples', 'oranges']

    def test_humanized_join(self):
        t = Template('{% load formatting %}{{ mylist|humanized_join }}')

        # with one item
        c = Context({"mylist": ['foo']})
        output = t.render(c)
        self.assertEqual(output, 'foo')

        # with two items
        c = Context({"mylist": self.test_list})
        output = t.render(c)
        self.assertEqual(output, 'apples and oranges')

        # with three
        self.test_list.append('pears')
        c = Context({"mylist": self.test_list})
        output = t.render(c)
        self.assertEqual(output, 'apples, oranges, and pears')

    def test_social_links(self):
        t = Template('{% load social_tags %}{% social_links object %}')
        c = Context({"object": ['foo']})
        output = t.render(c)
        self.assertTrue('facebook' in output)
        self.assertTrue('twitter' in output)
<commit_msg>using dummy model for test to get object<commit_after>import unittest

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.template import Template, Context
from django.test import TestCase

class DummyModel(models.Model):
    """
    Dummy model for testing below
    """
    def get_absolute_url(self):
        return '/'

class TestSharedContent(TestCase):

    def setUp(self):
        self.slug = 'admin'

    @unittest.skip("Makes multiple generous assumptions about project behaviour")
    def test_template_media(self):
        """
        Ensures base template has required media files.
        """
        response = self.client.get(reverse('home'))
        static_url = settings.STATIC_URL
        favicon_url = '<link rel="shortcut icon" href="{}img/favicon.png">'.format(static_url)
        touch_icon = '<link rel="apple-touch-icon" href="{}img/touch-icon.png">'.format(STATIC_URL)
        self.assertEqual(response.status_code, 200)
        self.assertTrue(favicon_url in response.content)
        self.assertTrue(touch_icon in response.content)

    @unittest.skip("Makes multiple generous assumptions about project behaviour")
    def test_shared_context_processor(self):
        """
        Test results of shared context processor are in template
        """
        response = self.client.get(reverse('home'))
        self.assertEqual(response.status_code, 200)
        self.assertTrue('site' in response.context)
        self.assertTrue('now' in response.context)
        self.assertTrue('year' in response.context)
        self.assertTrue('ga_code' in response.context)
        self.assertTrue('project_name' in response.context)
        self.assertTrue('current_path' in response.context)
        self.assertTrue('last_seen' in response.context)
        self.assertTrue('last_seen_fuzzy' in response.context)
        self.assertTrue('theme' in response.context)
        self.assertTrue('authenticated_request' in response.context)

class TemplateTagsTests(TestCase):
    def setUp(self):
        self.test_list = ['apples', 'oranges']

    def test_humanized_join(self):
        t = Template('{% load formatting %}{{ mylist|humanized_join }}')

        # with one item
        c = Context({"mylist": ['foo']})
        output = t.render(c)
        self.assertEqual(output, 'foo')

        # with two items
        c = Context({"mylist": self.test_list})
        output = t.render(c)
        self.assertEqual(output, 'apples and oranges')

        # with three
        self.test_list.append('pears')
        c = Context({"mylist": self.test_list})
        output = t.render(c)
        self.assertEqual(output, 'apples, oranges, and pears')

    def test_social_links(self):
        t = Template('{% load social_tags %}{% social_links object %}')
        obj = DummyModel.objects.create()
        c = Context({"object": obj})
        output = t.render(c)
        self.assertTrue('facebook' in output)
        self.assertTrue('twitter' in output)
<|endoftext|>


<commit_before>import re
import os
import os.path
import pathlib
import json
import hashlib
import random
import itertools
import lib.topologies as topologies
import lib.tests as tests
import lib.analyze as analyze
from lib.bash import CommandBlock
from pydblite import Base
results_dir = 'results/'

def generate_combinations(constants, variables, skip_fn=lambda x: False):
    n = 0
    variables_keys = tuple(variables.keys())
    for combi in itertools.product(*variables.values()):
        settings = constants.copy()
        for i, value in enumerate(combi):
            settings[variables_keys[i]] = value
        if not skip_fn(settings):
            generate(**settings)
            n += 1
    print('Generated {} cases. Now go and run run_test.py in order to run them!'.format(n))

def generate(**settings):
    settings_json = json.dumps(settings, sort_keys=True)
    settings_hash = hashlib.sha1(settings_json.encode('utf-8')).hexdigest()

    m, ns1, ns2 = getattr(topologies, settings['topology'])(**settings)
    settings['ns1'] = ns1
    settings['ns2'] = ns2

    settings['result_file'] = results_dir
    # if settings['collection'] is not None:
    #     settings['result_file'] += settings['collection'] + '/'
    settings['result_file'] += settings_hash

    script = tests.begin()
    script += m.get_script()
    script += getattr(tests, settings['iperf_name'])(**settings)

    with open(settings['result_file'] + '.config', 'w') as f:
        f.write(settings_json)
    with open(settings['result_file'] + '.sh', 'w') as f:
        f.write(str(script))
    os.chmod(settings['result_file'] + '.sh', 0o777)

    return settings_hash, script

def run_all(target_repetitions=0):
    to_run = []  # each script will appear N times, so to reach the target_repetitions
    p = pathlib.Path(results_dir)
    for script_file in p.glob('*.sh'):
        count = 0
        try:
            with script_file.parent.joinpath(script_file.stem + '.count').open() as count_fh:
                count = int(count_fh.read())
        except FileNotFoundError:
            pass
        to_run.extend([script_file] * (target_repetitions - count))
    random.shuffle(to_run)  # the order becomes unpredictable, because I think it's a good idea
    for current, script_file in enumerate(to_run, start=1):
        with script_file.open() as script_fh:
            script = CommandBlock() + script_fh.read()
            print("Running {} ({}/{})...".format(script_file.name, current, len(to_run)))
            script.run()

def get_results_db(clear_cache=False):
    cache_file = 'cache/results.pdl'
    db = Base(cache_file)

    if clear_cache or not db.exists() or os.path.getmtime(cache_file) < os.path.getmtime(results_dir):
        columns = set()
        rows = []
        p = pathlib.Path(results_dir)
        for config_file in p.glob('*.config'):
            with config_file.open() as config_fh:
                settings_hash = config_file.stem
                row = json.loads(config_fh.read())
            row['hash'] = settings_hash
            row['iostat_cpu'] = analyze.iostat_cpu(config_file.parent, settings_hash)
            row['iperf_result'] = getattr(analyze, row['iperf_name'])(config_file.parent, settings_hash, row)
            columns = columns | set(row.keys())
            rows.append(row)

        db.create(*columns, mode='override')
        for r in rows:
            db.insert(**r)
        db.commit()
    else:
        db.open()

    return db
<commit_msg>import style<commit_after>import re
import os
import os.path
import pathlib
import json
import hashlib
import random
import itertools
from . import topologies
from . import tests
from . import analyze
from lib.bash import CommandBlock
from pydblite import Base

results_dir = 'results/'

def generate_combinations(constants, variables, skip_fn=lambda x: False):
    n = 0
    variables_keys = tuple(variables.keys())
    for combi in itertools.product(*variables.values()):
        settings = constants.copy()
        for i, value in enumerate(combi):
            settings[variables_keys[i]] = value
        if not skip_fn(settings):
            generate(**settings)
            n += 1
    print('Generated {} cases. Now go and run run_test.py in order to run them!'.format(n))

def generate(**settings):
    settings_json = json.dumps(settings, sort_keys=True)
    settings_hash = hashlib.sha1(settings_json.encode('utf-8')).hexdigest()

    m, ns1, ns2 = getattr(topologies, settings['topology'])(**settings)
    settings['ns1'] = ns1
    settings['ns2'] = ns2

    settings['result_file'] = results_dir
    # if settings['collection'] is not None:
    #     settings['result_file'] += settings['collection'] + '/'
    settings['result_file'] += settings_hash

    script = tests.begin()
    script += m.get_script()
    script += getattr(tests, settings['iperf_name'])(**settings)

    with open(settings['result_file'] + '.config', 'w') as f:
        f.write(settings_json)
    with open(settings['result_file'] + '.sh', 'w') as f:
        f.write(str(script))
    os.chmod(settings['result_file'] + '.sh', 0o777)

    return settings_hash, script

def run_all(target_repetitions=0):
    to_run = []  # each script will appear N times, so to reach the target_repetitions
    p = pathlib.Path(results_dir)
    for script_file in p.glob('*.sh'):
        count = 0
        try:
            with script_file.parent.joinpath(script_file.stem + '.count').open() as count_fh:
                count = int(count_fh.read())
        except FileNotFoundError:
            pass
        to_run.extend([script_file] * (target_repetitions - count))
    random.shuffle(to_run)  # the order becomes unpredictable, because I think it's a good idea
    for current, script_file in enumerate(to_run, start=1):
        with script_file.open() as script_fh:
            script = CommandBlock() + script_fh.read()
            print("Running {} ({}/{})...".format(script_file.name, current, len(to_run)))
            script.run()

def get_results_db(clear_cache=False):
    cache_file = 'cache/results.pdl'
    db = Base(cache_file)

    if clear_cache or not db.exists() or os.path.getmtime(cache_file) < os.path.getmtime(results_dir):
        columns = set()
        rows = []
        p = pathlib.Path(results_dir)
        for config_file in p.glob('*.config'):
            with config_file.open() as config_fh:
                settings_hash = config_file.stem
                row = json.loads(config_fh.read())
            row['hash'] = settings_hash
            row['iostat_cpu'] = analyze.iostat_cpu(config_file.parent, settings_hash)
            row['iperf_result'] = getattr(analyze, row['iperf_name'])(config_file.parent, settings_hash, row)
            columns = columns | set(row.keys())
            rows.append(row)

        db.create(*columns, mode='override')
        for r in rows:
            db.insert(**r)
        db.commit()
    else:
        db.open()

    return db
<|endoftext|>


<commit_before><commit_msg>Another config import made safe<commit_after><|endoftext|>


<commit_before># Copyright 2019 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Cache handler - reads all the .dialog files (The default
mycroft responses) and does a tts inference.
It then saves the .wav files to mark1 device

"""

import base64
import glob
import os
import re
import shutil
import hashlib
import json
import mycroft.util as util
from urllib import parse
from requests_futures.sessions import FuturesSession
from mycroft.util.log import LOG

REGEX_SPL_CHARS = re.compile(r'[@#$%^*()<>/\|}{~:]')
MIMIC2_URL = 'https://mimic-api.mycroft.ai/synthesize?text='

# For now we only get the cache for mimic2-kusal
TTS = 'Mimic2'

# Check for more default dialogs
res_path = os.path.abspath(os.path.join(os.path.abspath(__file__), '..',
                                        '..', 'res', 'text', 'en-us'))
wifi_setup_path = '/usr/local/mycroft/mycroft-wifi-setup/dialog/en-us'
cache_dialog_path = [res_path, wifi_setup_path]

def generate_cache_text(cache_audio_dir, cache_text_file):
    """
    This prepares a text file with all the sentences
    from *.dialog files present in
    mycroft/res/text/en-us and mycroft-wifi setup skill
    Args:
        cache_audio_dir (path): DEPRECATED path to store .wav files
        cache_text_file (file): file containing the sentences
    """
    # TODO: remove in 21.08
    if cache_audio_dir is not None:
        LOG.warning(
            "the cache_audio_dir argument is deprecated. ensure the directory "
            "exists before executing this function. support for this argument "
            "will be removed in version 21.08"
        )
        if not os.path.exists(cache_audio_dir):
            os.makedirs(cache_audio_dir)
    try:
        if not os.path.isfile(cache_text_file):
            text_file = open(cache_text_file, 'w')
            for each_path in cache_dialog_path:
                if os.path.exists(each_path):
                    write_cache_text(each_path, text_file)
            text_file.close()
            LOG.debug("Completed generating cache")
        else:
            LOG.debug("Cache file 'cache_text.txt' already exists")
    except Exception:
        LOG.exception("Could not open text file to write cache")

def write_cache_text(cache_path, f):
    for file in glob.glob(cache_path + "/*.dialog"):
        try:
            with open(file, 'r') as fp:
                all_dialogs = fp.readlines()
                for each_dialog in all_dialogs:
                    # split the sentences
                    each_dialog = re.split(
                        r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\;|\?)\s',
                        each_dialog.strip())
                    for each in each_dialog:
                        if (REGEX_SPL_CHARS.search(each) is None):
                            # Do not consider sentences with special
                            # characters other than any punctuation
                            # ex : <<< LOADING <<<
                            # should not be considered
                            f.write(each.strip() + '\n')
        except Exception:
            # LOG.debug("Dialog Skipped")
            pass

def download_audio(cache_audio_dir, cache_text_file):
    """
    This method takes the sentences from the text file generated
    using generate_cache_text() and performs TTS inference on
    mimic2-api. The wav files and phonemes are stored in
    'cache_audio_dir'
    Args:
        cache_audio_dir (path): path to store .wav files
        cache_text_file (file): file containing the sentences
    """
    if os.path.isfile(cache_text_file) and \
            os.path.exists(cache_audio_dir):
        if not os.listdir(cache_audio_dir):
            session = FuturesSession()
            with open(cache_text_file, 'r') as fp:
                all_dialogs = fp.readlines()
                for each_dialog in all_dialogs:
                    each_dialog = each_dialog.strip()
                    key = str(hashlib.md5(
                        each_dialog.encode('utf-8', 'ignore')).hexdigest())
                    wav_file = os.path.join(cache_audio_dir, key + '.wav')
                    each_dialog = parse.quote(each_dialog)

                    mimic2_url = MIMIC2_URL + each_dialog + '&visimes=True'
                    try:
                        req = session.get(mimic2_url)
                        results = req.result().json()
                        audio = base64.b64decode(results['audio_base64'])
                        vis = results['visimes']
                        if audio:
                            with open(wav_file, 'wb') as audiofile:
                                audiofile.write(audio)
                        if vis:
                            pho_file = os.path.join(cache_audio_dir,
                                                    key + ".pho")
                            with open(pho_file, "w") as cachefile:
                                cachefile.write(json.dumps(vis))  # Mimic2
                                # cachefile.write(str(vis))  # Mimic
                    except Exception as e:
                        # Skip this dialog and continue
                        LOG.error("Unable to get pre-loaded cache "
                                  "due to ({})".format(repr(e)))

            LOG.debug("Completed getting cache for {}".format(TTS))

        else:
            LOG.debug("Pre-loaded cache for {} already exists".
                      format(TTS))
    else:
        missing_path = cache_text_file if not \
            os.path.isfile(cache_text_file)\
            else cache_audio_dir
        LOG.error("Path ({}) does not exist for getting the cache"
                  .format(missing_path))

def copy_cache(cache_audio_dir):
    """
    This method copies the cache from 'cache_audio_dir'
    to TTS specific cache directory given by
    get_cache_directory()
    Args:
        cache_audio_dir (path): path containing .wav files
    """
    if os.path.exists(cache_audio_dir):
        # get tmp directory where tts cache is stored
        dest = util.get_cache_directory('tts/' + 'Mimic2')
        files = os.listdir(cache_audio_dir)
        for f in files:
            shutil.copy2(os.path.join(cache_audio_dir, f), dest)
        LOG.debug("Copied all pre-loaded cache for {} to {}"
                  .format(TTS, dest))
    else:
        LOG.debug("No Source directory for {} pre-loaded cache"
                  .format(TTS))

# Start here
def main(cache_audio_dir):
    # Path where cache is stored and not cleared on reboot/TTS change
    if cache_audio_dir:
        if not os.path.exists(cache_audio_dir):
            os.makedirs(cache_audio_dir)
        cache_text_dir = os.path.dirname(cache_audio_dir)
        cache_text_path = os.path.join(cache_text_dir, 'cache_text.txt')
        # TODO: remove he first argument in 21.08
        generate_cache_text(None, cache_text_path)
        download_audio(cache_audio_dir, cache_text_path)
        copy_cache(cache_audio_dir)
<commit_msg>add deprecation warnings to the logic the new cache logic replaces.<commit_after># Copyright 2019 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Cache handler - reads all the .dialog files (The default
mycroft responses) and does a tts inference.
It then saves the .wav files to mark1 device

* * * *   D E P R E C A T E D   * * * *
THIS MODULE IS DEPRECATED IN FAVOR OF tts/cache.py. IT WILL BE REMOVED
IN THE NEXT MAJOR RELEASE, 21.08

"""
# TODO: remove in 21.08

import base64
import glob
import os
import re
import shutil
import hashlib
import json
import mycroft.util as util
from urllib import parse
from requests_futures.sessions import FuturesSession
from mycroft.util.log import LOG

REGEX_SPL_CHARS = re.compile(r'[@#$%^*()<>/\|}{~:]')
MIMIC2_URL = 'https://mimic-api.mycroft.ai/synthesize?text='

# For now we only get the cache for mimic2-kusal
TTS = 'Mimic2'

# Check for more default dialogs
res_path = os.path.abspath(os.path.join(os.path.abspath(__file__), '..',
                                        '..', 'res', 'text', 'en-us'))
wifi_setup_path = '/usr/local/mycroft/mycroft-wifi-setup/dialog/en-us'
cache_dialog_path = [res_path, wifi_setup_path]

def generate_cache_text(cache_audio_dir, cache_text_file):
    """
    This prepares a text file with all the sentences
    from *.dialog files present in
    mycroft/res/text/en-us and mycroft-wifi setup skill
    Args:
        cache_audio_dir (path): DEPRECATED path to store .wav files
        cache_text_file (file): file containing the sentences
    """
    # TODO: remove in 21.08
    if cache_audio_dir is not None:
        LOG.warning(
            "the cache_audio_dir argument is deprecated. ensure the directory "
            "exists before executing this function. support for this argument "
            "will be removed in version 21.08"
        )
        if not os.path.exists(cache_audio_dir):
            os.makedirs(cache_audio_dir)
    try:
        if not os.path.isfile(cache_text_file):
            text_file = open(cache_text_file, 'w')
            for each_path in cache_dialog_path:
                if os.path.exists(each_path):
                    write_cache_text(each_path, text_file)
            text_file.close()
            LOG.info("Completed generating cache")
        else:
            LOG.info("Cache file 'cache_text.txt' already exists")
    except Exception:
        LOG.exception("Could not open text file to write cache")

def write_cache_text(cache_path, f):
    # TODO: remove in 21.08
    for file in glob.glob(cache_path + "/*.dialog"):
        try:
            with open(file, 'r') as fp:
                all_dialogs = fp.readlines()
                for each_dialog in all_dialogs:
                    # split the sentences
                    each_dialog = re.split(
                        r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\;|\?)\s',
                        each_dialog.strip())
                    for each in each_dialog:
                        if (REGEX_SPL_CHARS.search(each) is None):
                            # Do not consider sentences with special
                            # characters other than any punctuation
                            # ex : <<< LOADING <<<
                            # should not be considered
                            f.write(each.strip() + '\n')
        except Exception:
            # LOG.debug("Dialog Skipped")
            pass

def download_audio(cache_audio_dir, cache_text_file):
    """
    This method takes the sentences from the text file generated
    using generate_cache_text() and performs TTS inference on
    mimic2-api. The wav files and phonemes are stored in
    'cache_audio_dir'
    Args:
        cache_audio_dir (path): path to store .wav files
        cache_text_file (file): file containing the sentences
    """
    # TODO: remove in 21.08
    if os.path.isfile(cache_text_file) and \
            os.path.exists(cache_audio_dir):
        if not os.listdir(cache_audio_dir):
            session = FuturesSession()
            with open(cache_text_file, 'r') as fp:
                all_dialogs = fp.readlines()
                for each_dialog in all_dialogs:
                    each_dialog = each_dialog.strip()
                    key = str(hashlib.md5(
                        each_dialog.encode('utf-8', 'ignore')).hexdigest())
                    wav_file = os.path.join(cache_audio_dir, key + '.wav')
                    each_dialog = parse.quote(each_dialog)

                    mimic2_url = MIMIC2_URL + each_dialog + '&visimes=True'
                    try:
                        req = session.get(mimic2_url)
                        results = req.result().json()
                        audio = base64.b64decode(results['audio_base64'])
                        vis = results['visimes']
                        if audio:
                            with open(wav_file, 'wb') as audiofile:
                                audiofile.write(audio)
                        if vis:
                            pho_file = os.path.join(cache_audio_dir,
                                                    key + ".pho")
                            with open(pho_file, "w") as cachefile:
                                cachefile.write(json.dumps(vis))  # Mimic2
                                # cachefile.write(str(vis))  # Mimic
                    except Exception:
                        # Skip this dialog and continue
                        LOG.exception("Unable to get pre-loaded cache")

            LOG.info("Completed getting cache for {}".format(TTS))

        else:
            LOG.info("Pre-loaded cache for {} already exists".format(TTS))
    else:
        missing_path = cache_text_file if not \
            os.path.isfile(cache_text_file)\
            else cache_audio_dir
        LOG.error("Path ({}) does not exist for getting the cache"
                  .format(missing_path))

def copy_cache(cache_audio_dir):
    """
    This method copies the cache from 'cache_audio_dir'
    to TTS specific cache directory given by
    get_cache_directory()
    Args:
        cache_audio_dir (path): path containing .wav files
    """
    # TODO: remove in 21.08
    if os.path.exists(cache_audio_dir):
        # get tmp directory where tts cache is stored
        dest = util.get_cache_directory('tts/' + 'Mimic2')
        files = os.listdir(cache_audio_dir)
        for f in files:
            shutil.copy2(os.path.join(cache_audio_dir, f), dest)
        LOG.info(
            "Copied all pre-loaded cache for {} to {}".format(TTS, dest))
    else:
        LOG.info(
            "No Source directory for {} pre-loaded cache".format(TTS))

# Start here
def main(cache_audio_dir):
    # TODO: remove in 21.08
    # Path where cache is stored and not cleared on reboot/TTS change
    if cache_audio_dir:
        if not os.path.exists(cache_audio_dir):
            os.makedirs(cache_audio_dir)
        cache_text_dir = os.path.dirname(cache_audio_dir)
        cache_text_path = os.path.join(cache_text_dir, 'cache_text.txt')
        # TODO: remove he first argument in 21.08
        generate_cache_text(None, cache_text_path)
        download_audio(cache_audio_dir, cache_text_path)
        copy_cache(cache_audio_dir)
<|endoftext|>


<commit_before>"""
Test creation of basic plot elements
"""
import numpy as np
import matplotlib.pyplot as plt
from .. import fig_to_dict
from numpy.testing import assert_equal

def test_line():
    fig, ax = plt.subplots()
    ax.plot(np.arange(10), np.random.random(10),
            '--k', alpha=0.3, zorder=10, lw=2)
    rep = fig_to_dict(fig)
    ax = rep['axes'][0]
    line = ax['lines'][0]

    assert_equal(list(sorted(line.keys())),
                 ['alpha', 'color', 'coordinates', 'dasharray', 'data', 'id',
                  'linewidth', 'xindex', 'yindex', 'zorder'])
    assert_equal(line['alpha'], 0.3)
    assert_equal(line['color'], "#000000")
    assert_equal(line['coordinates'], 'data')
    assert_equal(line['dasharray'], '6,6')
    assert_equal(line['zorder'], 10)
    assert_equal(line['linewidth'], 2)

def test_markers():
    fig, ax = plt.subplots()
    ax.plot(np.arange(10), np.random.random(10),
            '^k', alpha=0.3, zorder=10, mec='r', mew=2, c='b')
    rep = fig_to_dict(fig)
    ax = rep['axes'][0]
    markers = ax['markers'][0]

    assert_equal(list(sorted(markers.keys())),
                 ['alpha', 'coordinates', 'data', 'edgecolor', 'edgewidth',
                  'facecolor', 'id', 'markerpath', 'xindex', 'yindex',
                  'zorder'])
    assert_equal(markers['alpha'], 0.3)
    assert_equal(markers['zorder'], 10)
    assert_equal(markers['coordinates'], 'data')
    assert_equal(markers['edgecolor'], '#FF0000')
    assert_equal(markers['edgewidth'], 2)
    assert_equal(markers['facecolor'], '#0000FF')
    assert_equal(markers['markerpath'][0],
                 [[0.0, -3.0], [-3.0, 3.0], [3.0, 3.0]])
    assert_equal(markers['markerpath'][1],
                 ['M', 'L', 'L', 'Z'])

def test_scatter():
    fig, ax = plt.subplots()
    ax.scatter(np.arange(10), np.random.random(10), c='r', s=30,
               marker='^', alpha=0.3, lw=2, edgecolors='b', zorder=10)
    rep = fig_to_dict(fig)
    ax = rep['axes'][0]
    points = ax['collections'][0]

    assert_equal(list(sorted(points.keys())),
                 ['alphas', 'edgecolors', 'edgewidths', 'facecolors', 'id',
                  'offsetcoordinates', 'offsets', 'pathcoordinates', 'paths',
                  'pathtransforms', 'xindex', 'yindex', 'zorder'])
    assert_equal(points['alphas'], [0.3])
    assert_equal(points['zorder'], 10)
    assert_equal(points['edgecolors'], ['#0000FF'])
    assert_equal(points['facecolors'], ['#FF0000'])
    assert_equal(points['edgewidths'], (2.0,))
    assert_equal(points['paths'][0][0],
                 [[0.0, 0.5], [-0.5, -0.5], [0.5, -0.5]])
    assert_equal(points['paths'][0][1],
                 ['M', 'L', 'L', 'Z'])
    assert_equal(points['pathtransforms'],
                 [[6.085806194501846, 0.0, 0.0, 6.085806194501846, 0.0, 0.0]])
<commit_msg>TST: add tests for texts & paths<commit_after>"""
Test creation of basic plot elements
"""
import numpy as np
import matplotlib.pyplot as plt
from .. import fig_to_dict
from numpy.testing import assert_equal

def test_line():
    fig, ax = plt.subplots()
    ax.plot(np.arange(10), np.random.random(10),
            '--k', alpha=0.3, zorder=10, lw=2)
    rep = fig_to_dict(fig)
    axrep = rep['axes'][0]
    line = axrep['lines'][0]

    assert_equal(list(sorted(line.keys())),
                 ['alpha', 'color', 'coordinates', 'dasharray', 'data', 'id',
                  'linewidth', 'xindex', 'yindex', 'zorder'])
    assert_equal(line['alpha'], 0.3)
    assert_equal(line['color'], "#000000")
    assert_equal(line['coordinates'], 'data')
    assert_equal(line['dasharray'], '6,6')
    assert_equal(line['zorder'], 10)
    assert_equal(line['linewidth'], 2)

def test_markers():
    fig, ax = plt.subplots()
    ax.plot(np.arange(10), np.random.random(10),
            '^k', alpha=0.3, zorder=10, mec='r', mew=2, c='b')
    rep = fig_to_dict(fig)
    axrep = rep['axes'][0]
    markers = axrep['markers'][0]

    assert_equal(list(sorted(markers.keys())),
                 ['alpha', 'coordinates', 'data', 'edgecolor', 'edgewidth',
                  'facecolor', 'id', 'markerpath', 'xindex', 'yindex',
                  'zorder'])
    assert_equal(markers['alpha'], 0.3)
    assert_equal(markers['zorder'], 10)
    assert_equal(markers['coordinates'], 'data')
    assert_equal(markers['edgecolor'], '#FF0000')
    assert_equal(markers['edgewidth'], 2)
    assert_equal(markers['facecolor'], '#0000FF')
    assert_equal(markers['markerpath'][0],
                 [[0.0, -3.0], [-3.0, 3.0], [3.0, 3.0]])
    assert_equal(markers['markerpath'][1],
                 ['M', 'L', 'L', 'Z'])

def test_scatter():
    fig, ax = plt.subplots()
    ax.scatter(np.arange(10), np.random.random(10), c='r', s=30,
               marker='^', alpha=0.3, lw=2, edgecolors='b', zorder=10)
    rep = fig_to_dict(fig)
    axrep = rep['axes'][0]
    points = axrep['collections'][0]

    assert_equal(list(sorted(points.keys())),
                 ['alphas', 'edgecolors', 'edgewidths', 'facecolors', 'id',
                  'offsetcoordinates', 'offsets', 'pathcoordinates', 'paths',
                  'pathtransforms', 'xindex', 'yindex', 'zorder'])
    assert_equal(points['alphas'], [0.3])
    assert_equal(points['zorder'], 10)
    assert_equal(points['edgecolors'], ['#0000FF'])
    assert_equal(points['facecolors'], ['#FF0000'])
    assert_equal(points['edgewidths'], (2.0,))
    assert_equal(points['paths'][0][0],
                 [[0.0, 0.5], [-0.5, -0.5], [0.5, -0.5]])
    assert_equal(points['paths'][0][1],
                 ['M', 'L', 'L', 'Z'])
    assert_equal(points['pathtransforms'],
                 [[6.085806194501846, 0.0, 0.0, 6.085806194501846, 0.0, 0.0]])

def test_patch():
    fig, ax = plt.subplots()
    ax.add_patch(plt.Rectangle((0, 0), 1, 2, alpha=0.2, linewidth=2,
                               edgecolor='green', facecolor='red', zorder=3))
    rep = fig_to_dict(fig)
    axrep = rep['axes'][0]
    path = axrep['paths'][0]

    assert_equal(list(sorted(path.keys())),
                 ['alpha', 'coordinates', 'dasharray', 'data', 'edgecolor',
                  'edgewidth', 'facecolor', 'id', 'pathcodes',
                  'xindex', 'yindex', 'zorder'])

    assert_equal(path['alpha'], 0.2)
    assert_equal(path['edgecolor'], "#008000")
    assert_equal(path['facecolor'], "#FF0000")
    assert_equal(path['edgewidth'], 2)
    assert_equal(path['zorder'], 3)

def test_text():
    fig, ax = plt.subplots()
    ax.text(0.1, 0.1, "abcde", size=14, color='red', alpha=0.7,
            rotation=15, ha='center', va='center')
    rep = fig_to_dict(fig)
    axrep = rep['axes'][0]
    text = axrep['texts'][0]

    assert_equal(list(sorted(text.keys())),
                 ['alpha', 'color', 'coordinates', 'fontsize', 'h_anchor',
                  'id', 'position', 'rotation', 'text', 'v_baseline',
                  'zorder'])
    assert_equal(text['alpha'], 0.7)
    assert_equal(text['color'], "#FF0000")
    assert_equal(text['text'], "abcde")
    assert_equal(text['rotation'], -15)
    assert_equal(text['fontsize'], 14)
    assert_equal(text['position'], [0.1, 0.1])
    assert_equal(text['h_anchor'], 'middle')
    assert_equal(text['v_baseline'], 'central')
    assert_equal(text['zorder'], 3)
    assert_equal(text['coordinates'], "data")
<|endoftext|>


<commit_before>{
	"targets": [
		{
			"target_name": "node-libutp",
			'dependencies': [
				'libutp',
			],
			'include_dirs': [
				'deps/libutp',
				'<!(node -e "require(\'nan\')")',
			],
			"sources": [
				'src/utp_context.cc',
				'src/utp_socket.cc',
				'src/utp.cc'
			],
			'defines':[
				'UTP_DEBUG_LOGGING'
			],
			'conditions': [
				['OS=="win"', {
					'libraries': [
						'Ws2_32.lib'
					],
				}, { # OS!="win"
					'cflags': [
						'-std=c++14',
						'-fno-exceptions',
						'-g'
					],
					'defines':[
						'POSIX'
					],
					'libraries': [
					],
				}],
			]
		},
		{
			'target_name': 'libutp',
			'type': 'static_library',
			'defines': [
			    'UTP_DEBUG_LOGGING'
			],
			'sources': [
				'deps/libutp/utp_callbacks.cpp',
				'deps/libutp/utp_internal.cpp',
				'deps/libutp/utp_utils.cpp',
				'deps/libutp/utp_api.cpp',
				'deps/libutp/utp_hash.cpp',
				'deps/libutp/utp_packedsockaddr.cpp'
			],
			'conditions': [
				['OS=="linux" or OS=="android"', {
					'cxxflags': [
						'-Wall',
						'-fno-exceptions',
						'-fPIC',
						'-fno-rtti',
						'-Wno-sign-compare',
						'-fpermissive',
						'-g'
					],
					'defines' :[
						'POSIX',
					]
				}],
				['OS=="win"', {
					'defines': [
						'WIN32',
						'ENABLE_I18N',
						'ENABLE_SRP=1'
					],
					'sources':[
						'deps/libutp/libutp_inet_ntop.cpp',
					],
					'libraries': [
						'Ws2_32.lib'
					],
				}, {  # OS != "win",
				}],
			],
		},
	],
}
<commit_msg>lower c++ std<commit_after>{
	"targets": [
		{
			"target_name": "node-libutp",
			'dependencies': [
				'libutp',
			],
			'include_dirs': [
				'deps/libutp',
				'<!(node -e "require(\'nan\')")',
			],
			"sources": [
				'src/utp_context.cc',
				'src/utp_socket.cc',
				'src/utp.cc'
			],
			'defines':[
				'UTP_DEBUG_LOGGING'
			],
			'conditions': [
				['OS=="win"', {
					'libraries': [
						'Ws2_32.lib'
					],
				}, { # OS!="win"
					'cflags': [
						'-std=c++11',
						'-fno-exceptions',
						'-g'
					],
					'defines':[
						'POSIX'
					],
					'libraries': [
					],
				}],
			]
		},
		{
			'target_name': 'libutp',
			'type': 'static_library',
			'defines': [
			    'UTP_DEBUG_LOGGING'
			],
			'sources': [
				'deps/libutp/utp_callbacks.cpp',
				'deps/libutp/utp_internal.cpp',
				'deps/libutp/utp_utils.cpp',
				'deps/libutp/utp_api.cpp',
				'deps/libutp/utp_hash.cpp',
				'deps/libutp/utp_packedsockaddr.cpp'
			],
			'conditions': [
				['OS=="linux" or OS=="android"', {
					'cxxflags': [
						'-Wall',
						'-fno-exceptions',
						'-fPIC',
						'-fno-rtti',
						'-Wno-sign-compare',
						'-fpermissive',
						'-g'
					],
					'defines' :[
						'POSIX',
					]
				}],
				['OS=="win"', {
					'defines': [
						'WIN32',
						'ENABLE_I18N',
						'ENABLE_SRP=1'
					],
					'sources':[
						'deps/libutp/libutp_inet_ntop.cpp',
					],
					'libraries': [
						'Ws2_32.lib'
					],
				}, {  # OS != "win",
				}],
			],
		},
	],
}
<|endoftext|>