<commit_before>from datetime import datetime
from glob import glob
import locale
from pprint import pprint
import re
import sys
from textwrap import dedent

import mistune
from PIL import Image

ROOT = '/Users/karl/Sites/la-grange.net'
INDENTATION = re.compile(r'\n\s{2,}')
META = re.compile(r'^(\w+):\s*(.+?)\n')
PATH = re.compile(r'^.*(\d{4})/(\d{2})/(\d{2})/.*')
TEMPLATE = """date: {date}
prev: {prev}
title: {title}
url: {url}
style: /2019/style
"""

class GrangeRenderer(mistune.HTMLRenderer):
    """Adjusted renderer for La Grange."""
    def get_img_size(self, image_path):
        """extract width and height of an image."""
        full_path = ROOT + image_path
        try:
            with Image.open(full_path) as im:
                return im.size
        except FileNotFoundError as e:
            print('TOFIX: Image file path incorrect')
            sys.exit(f'       {e}')

    def image(self, src, alt="", title=None):
        width, height = self.get_img_size(src)
        if title:
            return dedent(f"""

{title}

                """)
        else:
            s = f''
            return s

    def paragraph(self, text):
        # In case of a figure, we do not want the (non-standard) paragraph.
        # david larlet's code idea
        if text.strip().startswith("
"):
            return text
        return f"
{text}
\n"

def parse(text):
    """Parse the given text into metadata and strip it for a Markdown parser.
    :param text: text to be parsed
    """
    rv = {}
    m = META.match(text)

    while m:
        key = m.group(1)
        value = m.group(2)
        value = INDENTATION.sub('\n', value.strip())
        rv[key] = value
        text = text[len(m.group(0)):]
        m = META.match(text)
    return rv, text

def main():
    locale.setlocale(locale.LC_ALL, 'fr_FR')
    import argparse
    from pathlib import Path

    parser = argparse.ArgumentParser()
    parser.add_argument("file_path", type=Path)

    p = parser.parse_args()
    entry_path = p.file_path
    # explore_path = ROOT + '/2019/*/*/*.md'
    # paths = glob(explore_path)
    # paths.sort()
    with open('/Users/karl/Sites/la-grange.net/2019/12/04/article_tmpl.html') as tmpfile:
        blog_tmp = tmpfile.read()
    # prev_title = 'Friction'
    # for entry_path in paths:

    with open(entry_path) as entry:
        text = entry.read()
        meta, entry_text = parse(text)
    pprint(meta)
    prev_url = meta['prev']
    with open(ROOT + prev_url + '.html') as prev_entry:
        from bs4 import BeautifulSoup
        text_prev = prev_entry.read()
        htmldata = BeautifulSoup(text_prev, features="lxml")
        prev_title = htmldata.find('title').text
        prev_title = prev_title.replace(' - Carnets Web de La Grange', '')

    # Meta extraction
    # Created
    created_timestamp = '{datestr}T23:59:59+09:00'.format(datestr=meta['date'])
    d = datetime.fromisoformat(meta['date'])
    day = d.day
    day_path = f"{d:%d}"
    year = d.year
    month = f"{d:%m}"
    month_name = f"{d:%B}"
    # special rendering
    renderer = GrangeRenderer()
    markdown = mistune.create_markdown(
        renderer=renderer, plugins=['strikethrough'])
    # metadata
    metadata = {
        'title': meta['title'],
        'created_timestamp': created_timestamp,
        'day': day,
        'year': year,
        'month': month,
        'month_name': month_name,
        'updated_timestamp': created_timestamp,
        'updated': meta['date'],
        'prev_url': meta['prev'],
        'prev_title': prev_title,
        'post_text': markdown(entry_text),
        'day_path': day_path,
        'url': meta['url'],
        'stylepath': meta['style'],
        }
    # print(meta)
    blog_post = blog_tmp.format(**metadata)
    dest = ROOT + '/{year}/{month}/{day_path}/{url}.html'.format(**metadata)
    print(dest)
    with open(dest, 'w') as blogpost:
        blogpost.write(blog_post)

def extract_date(path):
    full_date = PATH.match(path)
    return '-'.join(full_date.groups())

if __name__ == "__main__":
    main()
<commit_msg>issue #19 - fixes style mistakes<commit_after>from datetime import datetime
from glob import glob
import locale
from pprint import pprint
import re
import sys
from textwrap import dedent

import mistune
from PIL import Image

ROOT = '/Users/karl/Sites/la-grange.net'
INDENTATION = re.compile(r'\n\s{2,}')
META = re.compile(r'^(\w+):\s*(.+?)\n')
PATH = re.compile(r'^.*(\d{4})/(\d{2})/(\d{2})/.*')
TEMPLATE = """date: {date}
prev: {prev}
title: {title}
url: {url}
style: /2019/style
"""

class GrangeRenderer(mistune.HTMLRenderer):
    """Adjusted renderer for La Grange."""
    def get_img_size(self, image_path):
        """extract width and height of an image."""
        full_path = ROOT + image_path
        try:
            with Image.open(full_path) as im:
                return im.size
        except FileNotFoundError as e:
            print('TOFIX: Image file path incorrect')
            sys.exit(f'       {e}')

    def image(self, src, alt="", title=None):
        width, height = self.get_img_size(src)
        if title:
            return dedent(f"""

{title}

                """)
        else:
            s = f''  # noqa
            return s

    def paragraph(self, text):
        # In case of a figure, we do not want the (non-standard) paragraph.
        # david larlet's code idea
        if text.strip().startswith("
"):
            return text
        return f"
{text}
\n"

def parse(text):
    """Parse the given text into metadata and strip it for a Markdown parser.
    :param text: text to be parsed
    """
    rv = {}
    m = META.match(text)

    while m:
        key = m.group(1)
        value = m.group(2)
        value = INDENTATION.sub('\n', value.strip())
        rv[key] = value
        text = text[len(m.group(0)):]
        m = META.match(text)
    return rv, text

def main():
    locale.setlocale(locale.LC_ALL, 'fr_FR')
    import argparse
    from pathlib import Path

    parser = argparse.ArgumentParser()
    parser.add_argument("file_path", type=Path)

    p = parser.parse_args()
    entry_path = p.file_path
    # explore_path = ROOT + '/2019/*/*/*.md'
    # paths = glob(explore_path)
    # paths.sort()
    template_path = f'{ROOT}/2019/12/04/article_tmpl.html'
    with open(template_path) as tmpfile:
        blog_tmp = tmpfile.read()

    with open(entry_path) as entry:
        text = entry.read()
        meta, entry_text = parse(text)
    pprint(meta)
    prev_url = meta['prev']
    with open(ROOT + prev_url + '.html') as prev_entry:
        from bs4 import BeautifulSoup
        text_prev = prev_entry.read()
        htmldata = BeautifulSoup(text_prev, features="lxml")
        prev_title = htmldata.find('title').text
        prev_title = prev_title.replace(' - Carnets Web de La Grange', '')
    # Meta extraction
    # Created
    created_timestamp = '{datestr}T23:59:59+09:00'.format(datestr=meta['date'])
    d = datetime.fromisoformat(meta['date'])
    day = d.day
    day_path = f"{d:%d}"
    year = d.year
    month = f"{d:%m}"
    month_name = f"{d:%B}"
    # special rendering
    renderer = GrangeRenderer()
    markdown = mistune.create_markdown(
        renderer=renderer, plugins=['strikethrough'])
    # metadata
    metadata = {
        'title': meta['title'],
        'created_timestamp': created_timestamp,
        'day': day,
        'year': year,
        'month': month,
        'month_name': month_name,
        'updated_timestamp': created_timestamp,
        'updated': meta['date'],
        'prev_url': meta['prev'],
        'prev_title': prev_title,
        'post_text': markdown(entry_text),
        'day_path': day_path,
        'url': meta['url'],
        'stylepath': meta['style'],
        }
    blog_post = blog_tmp.format(**metadata)
    dest = ROOT + '/{year}/{month}/{day_path}/{url}.html'.format(**metadata)
    print(dest)
    with open(dest, 'w') as blogpost:
        blogpost.write(blog_post)

def extract_date(path):
    full_date = PATH.match(path)
    return '-'.join(full_date.groups())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Name: Yubico Python Client
# Description: Python class for verifying Yubico One Time Passwords (OTPs).
#
# Author: Tomaz Muraus (http://www.tomaz.me)
# License: BSD
#
# Copyright (c) 2010-2013, Tomaž Muraus
# Copyright (c) 2012, Yubico AB
# All rights reserved.

import re
import os
import sys
import time
import hmac
import base64
import hashlib
import threading
import logging

import requests

from yubico_client.otp import OTP
from yubico_client.yubico_exceptions import (StatusCodeError,
                                             InvalidClientIdError,
                                             InvalidValidationResponse,
                                             SignatureVerificationError)
from yubico_client.py3 import b
from yubico_client.py3 import urlencode

logger = logging.getLogger('yubico.client')

# Path to the custom CA certificates bundle. Only used if set.
CA_CERTS_BUNDLE_PATH = None

COMMON_CA_LOCATIONS = [
    '/usr/local/lib/ssl/certs/ca-certificates.crt',
    '/usr/local/ssl/certs/ca-certificates.crt',
    '/usr/local/share/curl/curl-ca-bundle.crt',
    '/usr/local/etc/openssl/cert.pem',
    '/opt/local/lib/ssl/certs/ca-certificates.crt',
    '/opt/local/ssl/certs/ca-certificates.crt',
    '/opt/local/share/curl/curl-ca-bundle.crt',
    '/opt/local/etc/openssl/cert.pem',
    '/usr/lib/ssl/certs/ca-certificates.crt',
    '/usr/ssl/certs/ca-certificates.crt',
    '/usr/share/curl/curl-ca-bundle.crt',
    '/etc/ssl/certs/ca-certificates.crt',
    '/etc/pki/tls/cert.pem',
    '/etc/pki/CA/cacert.pem',
    'C:\Windows\curl-ca-bundle.crt',
    'C:\Windows\ca-bundle.crt',
    'C:\Windows\cacert.pem'
]

DEFAULT_API_URLS = ('api.yubico.com/wsapi/2.0/verify',
                    'api2.yubico.com/wsapi/2.0/verify',
                    'api3.yubico.com/wsapi/2.0/verify',
                    'api4.yubico.com/wsapi/2.0/verify',
                    'api5.yubico.com/wsapi/2.0/verify')

DEFAULT_TIMEOUT = 10            # How long to wait before the time out occurs
DEFAULT_MAX_TIME_WINDOW = 5     # How many seconds can pass between the first
                                # and last OTP generations so the OTP is
                                # still considered valid.

BAD_STATUS_CODES = ['BAD_OTP', 'REPLAYED_OTP', 'BAD_SIGNATURE',
                    'MISSING_PARAMETER', 'OPERATION_NOT_ALLOWED',
                    'BACKEND_ERROR', 'NOT_ENOUGH_ANSWERS',
                    'REPLAYED_REQUEST']

class Yubico(object):
    def __init__(self, client_id, key=None, use_https=True, verify_cert=True,
                 translate_otp=True, api_urls=DEFAULT_API_URLS):
        self.client_id = client_id
        self.key = base64.b64decode(key) if key is not None else None
        self.verify_cert = verify_cert
        self.translate_otp = translate_otp
        self.api_urls = self._init_request_urls(api_urls, use_https)

    def verify(self, otp, timestamp=False, sl=None, timeout=None,
               return_response=False):
        """
        Verify a provided OTP.

        :param otp: OTP to verify.
        :type otp: ``str``

        :param timestamp: True to include request timestamp and session counter
                          in the response. Defaults to False.
        :type timestamp: ``bool``

        :param sl: A value indicating percentage of syncing required by client.
        :type sl: ``int`` or ``str``

        :param timeout: Number of seconds to wait for sync responses.
        :type timeout: ``int``

        :param return_response: True to return a response object instead of the
                                status code. Defaults to False.
        :type return_response: ``bool``

        :return: True is the provided OTP is valid, False if the
        REPLAYED_OTP status value is returned or the response message signature
        verification failed and None for the rest of the status values.
        """
        ca_bundle_path = self._get_ca_bundle_path()

        otp = OTP(otp, self.translate_otp)
        rand_str = b(os.urandom(30))
        nonce = base64.b64encode(rand_str, b('xz'))[:25]
        query_string = self.generate_query_string(otp.otp, nonce, timestamp,
                                                  sl, timeout)

        threads = []
        timeout = timeout or DEFAULT_TIMEOUT
        for url in self.api_urls:
            thread = URLThread('%s?%s' % (url, query_string), timeout,
                               self.verify_cert, ca_bundle_path)
            thread.start()
            threads.append(thread)

        # Wait for a first positive or negative response
        start_time = time.time()
        while threads and (start_time + timeout) > time.time():
            for thread in threads:
                if not thread.is_alive():
                    if thread.exception:
                        raise thread.exception
                    elif thread.response:
                        status = self.verify_response(thread.response,
                                                      otp.otp, nonce,
                                                      return_response)

                        if status:
                            if return_response:
                                return status
                            else:
                                return True
                    threads.remove(thread)
            time.sleep(0.1)

        # Timeout or no valid response received
        raise Exception('NO_VALID_ANSWERS')

    def verify_multi(self, otp_list, max_time_window=DEFAULT_MAX_TIME_WINDOW,
                     sl=None, timeout=None):
        """
        Verify a provided list of OTPs.

        :param max_time_window: Maximum number of seconds which can pass
                                between the first and last OTP generation for
                                the OTP to still be considered valid.
        :type max_time_window: ``int``
        """

        # Create the OTP objects
        otps = []
        for otp in otp_list:
            otps.append(OTP(otp, self.translate_otp))

        if len(otp_list) < 2:
            raise ValueError('otp_list needs to contain at least two OTPs')

        device_ids = set()
        for otp in otps:
            device_ids.add(otp.device_id)

        # Check that all the OTPs contain same device id
        if len(device_ids) != 1:
            raise Exception('OTPs contain different device ids')

        # Now we verify the OTPs and save the server response for each OTP.
        # We need the server response, to retrieve the timestamp.
        # It's possible to retrieve this value locally, without querying the
        # server but in this case, user would need to provide his AES key.
        for otp in otps:
            response = self.verify(otp.otp, True, sl, timeout,
                                   return_response=True)

            if not response:
                return False

            otp.timestamp = int(response['timestamp'])

        count = len(otps)
        delta = otps[count - 1].timestamp - otps[0].timestamp

        # OTPs have an 8Hz timestamp counter so we need to divide it to get
        # seconds
        delta = delta / 8

        if delta < 0:
            raise Exception('delta is smaller than zero. First OTP appears to '
                            'be older than the last one')

        if delta > max_time_window:
            raise Exception('More then %s seconds has passed between ' +
                            'generating the first and the last OTP.' %
                            (max_time_window * 0.125))

        return True

    def verify_response(self, response, otp, nonce, return_response=False):
        """
        Returns True if the OTP is valid (status=OK) and return_response=False,
        otherwise (return_response = True) it returns the server response as a
        dictionary.

        Throws an exception if the OTP is replayed, the server response message
        verification failed or the client id is invalid, returns False
        otherwise.
        """
        try:
            status = re.search(r'status=([A-Z0-9_]+)', response) \
                       .groups()

            if len(status) > 1:
                message = 'More than one status= returned. Possible attack!'
                raise InvalidValidationResponse(message, response)

            status = status[0]
        except (AttributeError, IndexError):
            return False

        signature, parameters = \
            self.parse_parameters_from_response(response)

        # Secret key is specified, so we verify the response message
        # signature
        if self.key:
            generated_signature = \
                self.generate_message_signature(parameters)

            # Signature located in the response does not match the one we
            # have generated
            if signature != generated_signature:
                raise SignatureVerificationError(generated_signature,
                                                 signature)
        param_dict = self.get_parameters_as_dictionary(parameters)

        if 'otp' in param_dict and param_dict['otp'] != otp:
            message = 'Unexpected OTP in response. Possible attack!'
            raise InvalidValidationResponse(message, response, param_dict)

        if 'nonce' in param_dict and param_dict['nonce'] != nonce:
            message = 'Unexpected nonce in response. Possible attack!'
            raise InvalidValidationResponse(message, response, param_dict)

        if status == 'OK':
            if return_response:
                return param_dict
            else:
                return True
        elif status == 'NO_SUCH_CLIENT':
            raise InvalidClientIdError(self.client_id)
        elif status == 'REPLAYED_OTP':
            raise StatusCodeError(status)

        return False

    def generate_query_string(self, otp, nonce, timestamp=False, sl=None,
                              timeout=None):
        """
        Returns a query string which is sent to the validation servers.
        """
        data = [('id', self.client_id),
                ('otp', otp),
                ('nonce', nonce)]

        if timestamp:
            data.append(('timestamp', '1'))

        if sl is not None:
            if sl not in range(0, 101) and sl not in ['fast', 'secure']:
                raise Exception('sl parameter value must be between 0 and '
                                '100 or string "fast" or "secure"')

            data.append(('sl', sl))

        if timeout:
            data.append(('timeout', timeout))

        query_string = urlencode(data)

        if self.key:
            hmac_signature = self.generate_message_signature(query_string)
            hmac_signature = hmac_signature
            query_string += '&h=%s' % (hmac_signature.replace('+', '%2B'))

        return query_string

    def generate_message_signature(self, query_string):
        """
        Returns a HMAC-SHA-1 signature for the given query string.
        http://goo.gl/R4O0E
        """
        pairs = query_string.split('&')
        pairs = [pair.split('=') for pair in pairs]
        pairs_sorted = sorted(pairs)
        pairs_string = '&' . join(['=' . join(pair) for pair in pairs_sorted])

        digest = hmac.new(self.key, b(pairs_string), hashlib.sha1).digest()
        signature = base64.b64encode(digest).decode('utf-8')

        return signature

    def parse_parameters_from_response(self, response):
        """
        Returns a response signature and query string generated from the
        server response. 'h' aka signature argument is stripped from the
        returned query string.
        """
        split = [pair.strip() for pair in response.split('\n')
                 if pair.strip() != '']
        query_string = '&' . join(split)
        split_dict = self.get_parameters_as_dictionary(query_string)

        if 'h' in split_dict:
            signature = split_dict['h']
            del split_dict['h']
        else:
            signature = None

        query_string = ''
        for index, (key, value) in enumerate(split_dict.items()):
            query_string += '%s=%s' % (key, value)

            if index != len(split_dict) - 1:
                query_string += '&'

        return (signature, query_string)

    def get_parameters_as_dictionary(self, query_string):
        """ Returns query string parameters as a dictionary. """
        dictionary = dict([parameter.split('=', 1) for parameter
                           in query_string.split('&')])
        return dictionary

    def _init_request_urls(self, api_urls, use_https=True):
        """
        Returns a list of the API URLs.
        """
        if not isinstance(api_urls, (str, list, tuple)):
            raise TypeError('api_urls needs to be string or iterable!')

        if isinstance(api_urls, str):
            api_urls = (api_urls,)

        urls = []
        for url in api_urls:
            if not url.startswith('http'):
                if use_https:
                    url = 'https://%s' % (url)
                else:
                    url = 'http://%s' % (url)
            urls.append(url)

        return urls

    def _get_ca_bundle_path(self):
        """
        Return a path to the CA bundle which is used for verifying the hosts
        SSL certificate.
        """
        if CA_CERTS_BUNDLE_PATH:
            # User provided a custom path
            return CA_CERTS_BUNDLE_PATH

        for file_path in COMMON_CA_LOCATIONS:
            if os.path.exists(file_path) and os.path.isfile(file_path):
                return file_path

        return None

class URLThread(threading.Thread):
    def __init__(self, url, timeout, verify_cert, ca_bundle_path=None):
        super(URLThread, self).__init__()
        self.url = url
        self.timeout = timeout
        self.verify_cert = verify_cert
        self.ca_bundle_path = ca_bundle_path
        self.exception = None
        self.request = None
        self.response = None

    def run(self):
        logger.debug('Sending HTTP request to %s (thread=%s)' % (self.url,
                                                                 self.name))
        verify = self.verify_cert

        if self.ca_bundle_path is not None:
            verify = self.ca_bundle_path
            logger.debug('Using custom CA bunde: %s' % (self.ca_bundle_path))

        try:
            self.request = requests.get(url=self.url, timeout=self.timeout,
                                        verify=verify)
            self.response = self.request.content.decode('utf-8')
        except requests.exceptions.SSLError:
            e = sys.exc_info()[1]
            self.exception = e
            self.response = None
        except Exception:
            e = sys.exc_info()[1]
            logger.error('Failed to retrieve response: ' + str(e))
            self.response = None

        args = (self.url, self.name, self.response)
        logger.debug('Received response from %s (thread=%s): %s' % args)
<commit_msg>Fix string interpolation.<commit_after># -*- coding: utf-8 -*-
#
# Name: Yubico Python Client
# Description: Python class for verifying Yubico One Time Passwords (OTPs).
#
# Author: Tomaz Muraus (http://www.tomaz.me)
# License: BSD
#
# Copyright (c) 2010-2013, Tomaž Muraus
# Copyright (c) 2012, Yubico AB
# All rights reserved.

import re
import os
import sys
import time
import hmac
import base64
import hashlib
import threading
import logging

import requests

from yubico_client.otp import OTP
from yubico_client.yubico_exceptions import (StatusCodeError,
                                             InvalidClientIdError,
                                             InvalidValidationResponse,
                                             SignatureVerificationError)
from yubico_client.py3 import b
from yubico_client.py3 import urlencode

logger = logging.getLogger('yubico.client')

# Path to the custom CA certificates bundle. Only used if set.
CA_CERTS_BUNDLE_PATH = None

COMMON_CA_LOCATIONS = [
    '/usr/local/lib/ssl/certs/ca-certificates.crt',
    '/usr/local/ssl/certs/ca-certificates.crt',
    '/usr/local/share/curl/curl-ca-bundle.crt',
    '/usr/local/etc/openssl/cert.pem',
    '/opt/local/lib/ssl/certs/ca-certificates.crt',
    '/opt/local/ssl/certs/ca-certificates.crt',
    '/opt/local/share/curl/curl-ca-bundle.crt',
    '/opt/local/etc/openssl/cert.pem',
    '/usr/lib/ssl/certs/ca-certificates.crt',
    '/usr/ssl/certs/ca-certificates.crt',
    '/usr/share/curl/curl-ca-bundle.crt',
    '/etc/ssl/certs/ca-certificates.crt',
    '/etc/pki/tls/cert.pem',
    '/etc/pki/CA/cacert.pem',
    'C:\Windows\curl-ca-bundle.crt',
    'C:\Windows\ca-bundle.crt',
    'C:\Windows\cacert.pem'
]

DEFAULT_API_URLS = ('api.yubico.com/wsapi/2.0/verify',
                    'api2.yubico.com/wsapi/2.0/verify',
                    'api3.yubico.com/wsapi/2.0/verify',
                    'api4.yubico.com/wsapi/2.0/verify',
                    'api5.yubico.com/wsapi/2.0/verify')

DEFAULT_TIMEOUT = 10            # How long to wait before the time out occurs
DEFAULT_MAX_TIME_WINDOW = 5     # How many seconds can pass between the first
                                # and last OTP generations so the OTP is
                                # still considered valid.

BAD_STATUS_CODES = ['BAD_OTP', 'REPLAYED_OTP', 'BAD_SIGNATURE',
                    'MISSING_PARAMETER', 'OPERATION_NOT_ALLOWED',
                    'BACKEND_ERROR', 'NOT_ENOUGH_ANSWERS',
                    'REPLAYED_REQUEST']

class Yubico(object):
    def __init__(self, client_id, key=None, use_https=True, verify_cert=True,
                 translate_otp=True, api_urls=DEFAULT_API_URLS):
        self.client_id = client_id
        self.key = base64.b64decode(key) if key is not None else None
        self.verify_cert = verify_cert
        self.translate_otp = translate_otp
        self.api_urls = self._init_request_urls(api_urls, use_https)

    def verify(self, otp, timestamp=False, sl=None, timeout=None,
               return_response=False):
        """
        Verify a provided OTP.

        :param otp: OTP to verify.
        :type otp: ``str``

        :param timestamp: True to include request timestamp and session counter
                          in the response. Defaults to False.
        :type timestamp: ``bool``

        :param sl: A value indicating percentage of syncing required by client.
        :type sl: ``int`` or ``str``

        :param timeout: Number of seconds to wait for sync responses.
        :type timeout: ``int``

        :param return_response: True to return a response object instead of the
                                status code. Defaults to False.
        :type return_response: ``bool``

        :return: True is the provided OTP is valid, False if the
        REPLAYED_OTP status value is returned or the response message signature
        verification failed and None for the rest of the status values.
        """
        ca_bundle_path = self._get_ca_bundle_path()

        otp = OTP(otp, self.translate_otp)
        rand_str = b(os.urandom(30))
        nonce = base64.b64encode(rand_str, b('xz'))[:25]
        query_string = self.generate_query_string(otp.otp, nonce, timestamp,
                                                  sl, timeout)

        threads = []
        timeout = timeout or DEFAULT_TIMEOUT
        for url in self.api_urls:
            thread = URLThread('%s?%s' % (url, query_string), timeout,
                               self.verify_cert, ca_bundle_path)
            thread.start()
            threads.append(thread)

        # Wait for a first positive or negative response
        start_time = time.time()
        while threads and (start_time + timeout) > time.time():
            for thread in threads:
                if not thread.is_alive():
                    if thread.exception:
                        raise thread.exception
                    elif thread.response:
                        status = self.verify_response(thread.response,
                                                      otp.otp, nonce,
                                                      return_response)

                        if status:
                            if return_response:
                                return status
                            else:
                                return True
                    threads.remove(thread)
            time.sleep(0.1)

        # Timeout or no valid response received
        raise Exception('NO_VALID_ANSWERS')

    def verify_multi(self, otp_list, max_time_window=DEFAULT_MAX_TIME_WINDOW,
                     sl=None, timeout=None):
        """
        Verify a provided list of OTPs.

        :param max_time_window: Maximum number of seconds which can pass
                                between the first and last OTP generation for
                                the OTP to still be considered valid.
        :type max_time_window: ``int``
        """

        # Create the OTP objects
        otps = []
        for otp in otp_list:
            otps.append(OTP(otp, self.translate_otp))

        if len(otp_list) < 2:
            raise ValueError('otp_list needs to contain at least two OTPs')

        device_ids = set()
        for otp in otps:
            device_ids.add(otp.device_id)

        # Check that all the OTPs contain same device id
        if len(device_ids) != 1:
            raise Exception('OTPs contain different device ids')

        # Now we verify the OTPs and save the server response for each OTP.
        # We need the server response, to retrieve the timestamp.
        # It's possible to retrieve this value locally, without querying the
        # server but in this case, user would need to provide his AES key.
        for otp in otps:
            response = self.verify(otp.otp, True, sl, timeout,
                                   return_response=True)

            if not response:
                return False

            otp.timestamp = int(response['timestamp'])

        count = len(otps)
        delta = otps[count - 1].timestamp - otps[0].timestamp

        # OTPs have an 8Hz timestamp counter so we need to divide it to get
        # seconds
        delta = delta / 8

        if delta < 0:
            raise Exception('delta is smaller than zero. First OTP appears to '
                            'be older than the last one')

        if delta > max_time_window:
            raise Exception(('More then %s seconds has passed between '
                            'generating the first and the last OTP.') %
                            (max_time_window))

        return True

    def verify_response(self, response, otp, nonce, return_response=False):
        """
        Returns True if the OTP is valid (status=OK) and return_response=False,
        otherwise (return_response = True) it returns the server response as a
        dictionary.

        Throws an exception if the OTP is replayed, the server response message
        verification failed or the client id is invalid, returns False
        otherwise.
        """
        try:
            status = re.search(r'status=([A-Z0-9_]+)', response) \
                       .groups()

            if len(status) > 1:
                message = 'More than one status= returned. Possible attack!'
                raise InvalidValidationResponse(message, response)

            status = status[0]
        except (AttributeError, IndexError):
            return False

        signature, parameters = \
            self.parse_parameters_from_response(response)

        # Secret key is specified, so we verify the response message
        # signature
        if self.key:
            generated_signature = \
                self.generate_message_signature(parameters)

            # Signature located in the response does not match the one we
            # have generated
            if signature != generated_signature:
                raise SignatureVerificationError(generated_signature,
                                                 signature)
        param_dict = self.get_parameters_as_dictionary(parameters)

        if 'otp' in param_dict and param_dict['otp'] != otp:
            message = 'Unexpected OTP in response. Possible attack!'
            raise InvalidValidationResponse(message, response, param_dict)

        if 'nonce' in param_dict and param_dict['nonce'] != nonce:
            message = 'Unexpected nonce in response. Possible attack!'
            raise InvalidValidationResponse(message, response, param_dict)

        if status == 'OK':
            if return_response:
                return param_dict
            else:
                return True
        elif status == 'NO_SUCH_CLIENT':
            raise InvalidClientIdError(self.client_id)
        elif status == 'REPLAYED_OTP':
            raise StatusCodeError(status)

        return False

    def generate_query_string(self, otp, nonce, timestamp=False, sl=None,
                              timeout=None):
        """
        Returns a query string which is sent to the validation servers.
        """
        data = [('id', self.client_id),
                ('otp', otp),
                ('nonce', nonce)]

        if timestamp:
            data.append(('timestamp', '1'))

        if sl is not None:
            if sl not in range(0, 101) and sl not in ['fast', 'secure']:
                raise Exception('sl parameter value must be between 0 and '
                                '100 or string "fast" or "secure"')

            data.append(('sl', sl))

        if timeout:
            data.append(('timeout', timeout))

        query_string = urlencode(data)

        if self.key:
            hmac_signature = self.generate_message_signature(query_string)
            hmac_signature = hmac_signature
            query_string += '&h=%s' % (hmac_signature.replace('+', '%2B'))

        return query_string

    def generate_message_signature(self, query_string):
        """
        Returns a HMAC-SHA-1 signature for the given query string.
        http://goo.gl/R4O0E
        """
        pairs = query_string.split('&')
        pairs = [pair.split('=') for pair in pairs]
        pairs_sorted = sorted(pairs)
        pairs_string = '&' . join(['=' . join(pair) for pair in pairs_sorted])

        digest = hmac.new(self.key, b(pairs_string), hashlib.sha1).digest()
        signature = base64.b64encode(digest).decode('utf-8')

        return signature

    def parse_parameters_from_response(self, response):
        """
        Returns a response signature and query string generated from the
        server response. 'h' aka signature argument is stripped from the
        returned query string.
        """
        split = [pair.strip() for pair in response.split('\n')
                 if pair.strip() != '']
        query_string = '&' . join(split)
        split_dict = self.get_parameters_as_dictionary(query_string)

        if 'h' in split_dict:
            signature = split_dict['h']
            del split_dict['h']
        else:
            signature = None

        query_string = ''
        for index, (key, value) in enumerate(split_dict.items()):
            query_string += '%s=%s' % (key, value)

            if index != len(split_dict) - 1:
                query_string += '&'

        return (signature, query_string)

    def get_parameters_as_dictionary(self, query_string):
        """ Returns query string parameters as a dictionary. """
        dictionary = dict([parameter.split('=', 1) for parameter
                           in query_string.split('&')])
        return dictionary

    def _init_request_urls(self, api_urls, use_https=True):
        """
        Returns a list of the API URLs.
        """
        if not isinstance(api_urls, (str, list, tuple)):
            raise TypeError('api_urls needs to be string or iterable!')

        if isinstance(api_urls, str):
            api_urls = (api_urls,)

        urls = []
        for url in api_urls:
            if not url.startswith('http'):
                if use_https:
                    url = 'https://%s' % (url)
                else:
                    url = 'http://%s' % (url)
            urls.append(url)

        return urls

    def _get_ca_bundle_path(self):
        """
        Return a path to the CA bundle which is used for verifying the hosts
        SSL certificate.
        """
        if CA_CERTS_BUNDLE_PATH:
            # User provided a custom path
            return CA_CERTS_BUNDLE_PATH

        for file_path in COMMON_CA_LOCATIONS:
            if os.path.exists(file_path) and os.path.isfile(file_path):
                return file_path

        return None

class URLThread(threading.Thread):
    def __init__(self, url, timeout, verify_cert, ca_bundle_path=None):
        super(URLThread, self).__init__()
        self.url = url
        self.timeout = timeout
        self.verify_cert = verify_cert
        self.ca_bundle_path = ca_bundle_path
        self.exception = None
        self.request = None
        self.response = None

    def run(self):
        logger.debug('Sending HTTP request to %s (thread=%s)' % (self.url,
                                                                 self.name))
        verify = self.verify_cert

        if self.ca_bundle_path is not None:
            verify = self.ca_bundle_path
            logger.debug('Using custom CA bunde: %s' % (self.ca_bundle_path))

        try:
            self.request = requests.get(url=self.url, timeout=self.timeout,
                                        verify=verify)
            self.response = self.request.content.decode('utf-8')
        except requests.exceptions.SSLError:
            e = sys.exc_info()[1]
            self.exception = e
            self.response = None
        except Exception:
            e = sys.exc_info()[1]
            logger.error('Failed to retrieve response: ' + str(e))
            self.response = None

        args = (self.url, self.name, self.response)
        logger.debug('Received response from %s (thread=%s): %s' % args)
<|endoftext|>


<commit_before>
from datetime import timedelta

from django.conf import settings
from django.db import connection, transaction
from django.utils.timezone import now as timezone_now
from zerver.models import (Message, UserMessage, ArchivedMessage, ArchivedUserMessage, Realm,
                           Attachment, ArchivedAttachment, Reaction, ArchivedReaction,
                           SubMessage, ArchivedSubMessage, Recipient, Stream,
                           get_stream_recipients, get_user_including_cross_realm)

from typing import Any, Dict, List

models_with_message_key = [
    {
        'class': Reaction,
        'archive_class': ArchivedReaction,
        'table_name': 'zerver_reaction',
        'archive_table_name': 'zerver_archivedreaction'
    },
    {
        'class': SubMessage,
        'archive_class': ArchivedSubMessage,
        'table_name': 'zerver_submessage',
        'archive_table_name': 'zerver_archivedsubmessage'
    },
]  # type: List[Dict[str, Any]]

@transaction.atomic
def move_rows(src_model: Any, raw_query: str, returning_id: bool=False,
              **kwargs: Any) -> List[int]:
    src_db_table = src_model._meta.db_table
    src_fields = ["{}.{}".format(src_db_table, field.column) for field in src_model._meta.fields]
    dst_fields = [field.column for field in src_model._meta.fields]
    sql_args = {
        'src_fields': ','.join(src_fields),
        'dst_fields': ','.join(dst_fields),
        'archive_timestamp': timezone_now()
    }
    sql_args.update(kwargs)
    with connection.cursor() as cursor:
        cursor.execute(
            raw_query.format(**sql_args)
        )
        if returning_id:
            return [row[0] for row in cursor.fetchall()]  # return list of row ids
        else:
            return []

def ids_list_to_sql_query_format(ids: List[int]) -> str:
    assert len(ids) > 0

    ids_tuple = tuple(ids)
    if len(ids_tuple) > 1:
        ids_string = str(ids_tuple)
    elif len(ids_tuple) == 1:
        ids_string = '({})'.format(ids_tuple[0])

    return ids_string

def move_expired_messages_to_archive_by_recipient(recipient: Recipient,
                                                  message_retention_days: int) -> List[int]:
    query = """
    INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_message
    LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
    WHERE zerver_message.recipient_id = {recipient_id}
        AND zerver_message.pub_date < '{check_date}'
        AND zerver_archivedmessage.id is NULL
    RETURNING id
    """
    check_date = timezone_now() - timedelta(days=message_retention_days)

    return move_rows(Message, query, returning_id=True,
                     recipient_id=recipient.id, check_date=check_date.isoformat())

def move_expired_personal_and_huddle_messages_to_archive(realm: Realm) -> List[int]:
    cross_realm_bot_ids_list = [get_user_including_cross_realm(email).id
                                for email in settings.CROSS_REALM_BOT_EMAILS]
    cross_realm_bot_ids = str(tuple(cross_realm_bot_ids_list))
    recipient_types = (Recipient.PERSONAL, Recipient.HUDDLE)

    # Archive expired personal and huddle Messages in the realm, except cross-realm messages:
    # TODO: Remove the "zerver_userprofile.id NOT IN {cross_realm_bot_ids}" clause
    # once https://github.com/zulip/zulip/issues/11015 is solved.
    query = """
    INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_message
    INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
    INNER JOIN zerver_userprofile ON zerver_userprofile.id = zerver_message.sender_id
    LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
    WHERE zerver_userprofile.id NOT IN {cross_realm_bot_ids}
        AND zerver_userprofile.realm_id = {realm_id}
        AND zerver_recipient.type in {recipient_types}
        AND zerver_message.pub_date < '{check_date}'
        AND zerver_archivedmessage.id is NULL
    RETURNING id
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)

    return move_rows(Message, query, returning_id=True, cross_realm_bot_ids=cross_realm_bot_ids,
                     realm_id=realm.id, recipient_types=recipient_types, check_date=check_date.isoformat())

def move_user_messages_to_archive(msg_ids: List[int]) -> List[int]:
    if not msg_ids:
        return []

    query = """
    INSERT INTO zerver_archivedusermessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_usermessage
    LEFT JOIN zerver_archivedusermessage ON zerver_archivedusermessage.id = zerver_usermessage.id
    WHERE zerver_usermessage.message_id IN {message_ids}
        AND zerver_archivedusermessage.id is NULL
    RETURNING id
    """

    return move_rows(UserMessage, query, returning_id=True,
                     message_ids=ids_list_to_sql_query_format(msg_ids))

def move_models_with_message_key_to_archive(msg_ids: List[int]) -> None:
    if not msg_ids:
        return

    for model in models_with_message_key:
        query = """
        INSERT INTO {archive_table_name} ({dst_fields}, archive_timestamp)
        SELECT {src_fields}, '{archive_timestamp}'
        FROM {table_name}
        LEFT JOIN {archive_table_name} ON {archive_table_name}.id = {table_name}.id
        WHERE {table_name}.message_id IN {message_ids}
            AND {archive_table_name}.id IS NULL
        """
        move_rows(model['class'], query, table_name=model['table_name'],
                  archive_table_name=model['archive_table_name'],
                  message_ids=ids_list_to_sql_query_format(msg_ids))

def move_attachments_to_archive(msg_ids: List[int]) -> None:
    if not msg_ids:
        return

    query = """
       INSERT INTO zerver_archivedattachment ({dst_fields}, archive_timestamp)
       SELECT {src_fields}, '{archive_timestamp}'
       FROM zerver_attachment
       INNER JOIN zerver_attachment_messages
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       LEFT JOIN zerver_archivedattachment ON zerver_archivedattachment.id = zerver_attachment.id
       WHERE zerver_attachment_messages.message_id IN {message_ids}
            AND zerver_archivedattachment.id IS NULL
       GROUP BY zerver_attachment.id
    """
    move_rows(Attachment, query, message_ids=ids_list_to_sql_query_format(msg_ids))

def move_attachments_message_rows_to_archive(msg_ids: List[int]) -> None:
    if not msg_ids:
        return

    query = """
       INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id, archivedmessage_id)
       SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
           zerver_attachment_messages.message_id
       FROM zerver_attachment_messages
       INNER JOIN zerver_attachment
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       LEFT JOIN zerver_archivedattachment_messages
           ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
       WHERE  zerver_attachment_messages.message_id IN {message_ids}
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    with connection.cursor() as cursor:
        cursor.execute(query.format(message_ids=ids_list_to_sql_query_format(msg_ids)))

def delete_messages(msg_ids: List[int]) -> None:
    Message.objects.filter(
        usermessage__isnull=True, id__in=msg_ids
    ).delete()

def delete_user_messages(usermsg_ids: List[int]) -> None:
    UserMessage.objects.filter(id__in=usermsg_ids).delete()

def delete_expired_attachments() -> None:
    Attachment.objects.filter(
        messages__isnull=True, id__in=ArchivedAttachment.objects.all(),
    ).delete()

def move_related_objects_to_archive(msg_ids: List[int]) -> List[int]:
    usermsg_ids = move_user_messages_to_archive(msg_ids)
    move_models_with_message_key_to_archive(msg_ids)
    move_attachments_to_archive(msg_ids)
    move_attachments_message_rows_to_archive(msg_ids)

    return usermsg_ids

def clean_expired(msg_ids: List[int], usermsg_ids: List[int]) -> None:
    delete_user_messages(usermsg_ids)
    delete_messages(msg_ids)

def archive_messages_by_recipient(recipient: Recipient, message_retention_days: int) -> None:
    msg_ids = move_expired_messages_to_archive_by_recipient(recipient, message_retention_days)
    usermsg_ids = move_related_objects_to_archive(msg_ids)
    clean_expired(msg_ids, usermsg_ids)

def archive_personal_and_huddle_messages() -> None:
    for realm in Realm.objects.filter(message_retention_days__isnull=False):
        msg_ids = move_expired_personal_and_huddle_messages_to_archive(realm)
        usermsg_ids = move_related_objects_to_archive(msg_ids)
        clean_expired(msg_ids, usermsg_ids)

def archive_stream_messages() -> None:
    streams = Stream.objects.filter(message_retention_days__isnull=False)
    retention_policy_dict = {}  # type: Dict[int, int]
    for stream in streams:
        retention_policy_dict[stream.id] = stream.message_retention_days

    recipients = get_stream_recipients([stream.id for stream in streams])
    for recipient in recipients:
        archive_messages_by_recipient(recipient, retention_policy_dict[recipient.type_id])

def archive_messages() -> None:
    archive_stream_messages()
    archive_personal_and_huddle_messages()

    # Since figuring out which attachments can be deleted requires scanning the whole
    # ArchivedAttachment table, we should do this just once, at the end of the archiving process:
    delete_expired_attachments()

def move_attachment_messages_to_archive_by_message(message_ids: List[int]) -> None:
    # Move attachments messages relation table data to archive.
    id_list = ', '.join(str(message_id) for message_id in message_ids)

    query = """
        INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id,
            archivedmessage_id)
        SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
            zerver_attachment_messages.message_id
        FROM zerver_attachment_messages
        LEFT JOIN zerver_archivedattachment_messages
            ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
        WHERE zerver_attachment_messages.message_id in ({message_ids})
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    with connection.cursor() as cursor:
        cursor.execute(query.format(message_ids=id_list))

@transaction.atomic
def move_messages_to_archive(message_ids: List[int]) -> None:
    messages = list(Message.objects.filter(id__in=message_ids).values())
    if not messages:
        raise Message.DoesNotExist

    ArchivedMessage.objects.bulk_create([ArchivedMessage(**message) for message in messages])

    # Move user_messages to the archive.
    user_messages = UserMessage.objects.filter(
        message_id__in=message_ids).exclude(id__in=ArchivedUserMessage.objects.all()).values()
    user_messages_ids = [user_message['id'] for user_message in user_messages]
    ArchivedUserMessage.objects.bulk_create(
        [ArchivedUserMessage(**user_message) for user_message in user_messages]
    )

    for model in models_with_message_key:
        elements = model['class'].objects.filter(message_id__in=message_ids).exclude(
            id__in=model['archive_class'].objects.all()
        ).values()

        model['archive_class'].objects.bulk_create(
            [model['archive_class'](**element) for element in elements]
        )

    # Move attachments to archive
    attachments = Attachment.objects.filter(messages__id__in=message_ids).exclude(
        id__in=ArchivedAttachment.objects.all()).distinct().values()
    ArchivedAttachment.objects.bulk_create(
        [ArchivedAttachment(**attachment) for attachment in attachments]
    )

    move_attachment_messages_to_archive_by_message(message_ids)

    # Remove data from main tables
    Message.objects.filter(id__in=message_ids).delete()
    UserMessage.objects.filter(id__in=user_messages_ids, message_id__isnull=True).delete()

    archived_attachments = ArchivedAttachment.objects.filter(messages__id__in=message_ids).distinct()
    Attachment.objects.filter(messages__isnull=True, id__in=archived_attachments).delete()
<commit_msg>retention: Clean up redundant code for special handling of UserMessages.<commit_after>
from datetime import timedelta

from django.conf import settings
from django.db import connection, transaction
from django.utils.timezone import now as timezone_now
from zerver.models import (Message, UserMessage, ArchivedMessage, ArchivedUserMessage, Realm,
                           Attachment, ArchivedAttachment, Reaction, ArchivedReaction,
                           SubMessage, ArchivedSubMessage, Recipient, Stream,
                           get_stream_recipients, get_user_including_cross_realm)

from typing import Any, Dict, List

models_with_message_key = [
    {
        'class': Reaction,
        'archive_class': ArchivedReaction,
        'table_name': 'zerver_reaction',
        'archive_table_name': 'zerver_archivedreaction'
    },
    {
        'class': SubMessage,
        'archive_class': ArchivedSubMessage,
        'table_name': 'zerver_submessage',
        'archive_table_name': 'zerver_archivedsubmessage'
    },
    {
        'class': UserMessage,
        'archive_class': ArchivedUserMessage,
        'table_name': 'zerver_usermessage',
        'archive_table_name': 'zerver_archivedusermessage'
    },
]  # type: List[Dict[str, Any]]

@transaction.atomic
def move_rows(src_model: Any, raw_query: str, returning_id: bool=False,
              **kwargs: Any) -> List[int]:
    src_db_table = src_model._meta.db_table
    src_fields = ["{}.{}".format(src_db_table, field.column) for field in src_model._meta.fields]
    dst_fields = [field.column for field in src_model._meta.fields]
    sql_args = {
        'src_fields': ','.join(src_fields),
        'dst_fields': ','.join(dst_fields),
        'archive_timestamp': timezone_now()
    }
    sql_args.update(kwargs)
    with connection.cursor() as cursor:
        cursor.execute(
            raw_query.format(**sql_args)
        )
        if returning_id:
            return [row[0] for row in cursor.fetchall()]  # return list of row ids
        else:
            return []

def ids_list_to_sql_query_format(ids: List[int]) -> str:
    assert len(ids) > 0

    ids_tuple = tuple(ids)
    if len(ids_tuple) > 1:
        ids_string = str(ids_tuple)
    elif len(ids_tuple) == 1:
        ids_string = '({})'.format(ids_tuple[0])

    return ids_string

def move_expired_messages_to_archive_by_recipient(recipient: Recipient,
                                                  message_retention_days: int) -> List[int]:
    query = """
    INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_message
    LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
    WHERE zerver_message.recipient_id = {recipient_id}
        AND zerver_message.pub_date < '{check_date}'
        AND zerver_archivedmessage.id is NULL
    RETURNING id
    """
    check_date = timezone_now() - timedelta(days=message_retention_days)

    return move_rows(Message, query, returning_id=True,
                     recipient_id=recipient.id, check_date=check_date.isoformat())

def move_expired_personal_and_huddle_messages_to_archive(realm: Realm) -> List[int]:
    cross_realm_bot_ids_list = [get_user_including_cross_realm(email).id
                                for email in settings.CROSS_REALM_BOT_EMAILS]
    cross_realm_bot_ids = str(tuple(cross_realm_bot_ids_list))
    recipient_types = (Recipient.PERSONAL, Recipient.HUDDLE)

    # Archive expired personal and huddle Messages in the realm, except cross-realm messages:
    # TODO: Remove the "zerver_userprofile.id NOT IN {cross_realm_bot_ids}" clause
    # once https://github.com/zulip/zulip/issues/11015 is solved.
    query = """
    INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_message
    INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
    INNER JOIN zerver_userprofile ON zerver_userprofile.id = zerver_message.sender_id
    LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
    WHERE zerver_userprofile.id NOT IN {cross_realm_bot_ids}
        AND zerver_userprofile.realm_id = {realm_id}
        AND zerver_recipient.type in {recipient_types}
        AND zerver_message.pub_date < '{check_date}'
        AND zerver_archivedmessage.id is NULL
    RETURNING id
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)

    return move_rows(Message, query, returning_id=True, cross_realm_bot_ids=cross_realm_bot_ids,
                     realm_id=realm.id, recipient_types=recipient_types, check_date=check_date.isoformat())

def move_models_with_message_key_to_archive(msg_ids: List[int]) -> None:
    if not msg_ids:
        return

    for model in models_with_message_key:
        query = """
        INSERT INTO {archive_table_name} ({dst_fields}, archive_timestamp)
        SELECT {src_fields}, '{archive_timestamp}'
        FROM {table_name}
        LEFT JOIN {archive_table_name} ON {archive_table_name}.id = {table_name}.id
        WHERE {table_name}.message_id IN {message_ids}
            AND {archive_table_name}.id IS NULL
        """
        move_rows(model['class'], query, table_name=model['table_name'],
                  archive_table_name=model['archive_table_name'],
                  message_ids=ids_list_to_sql_query_format(msg_ids))

def move_attachments_to_archive(msg_ids: List[int]) -> None:
    if not msg_ids:
        return

    query = """
       INSERT INTO zerver_archivedattachment ({dst_fields}, archive_timestamp)
       SELECT {src_fields}, '{archive_timestamp}'
       FROM zerver_attachment
       INNER JOIN zerver_attachment_messages
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       LEFT JOIN zerver_archivedattachment ON zerver_archivedattachment.id = zerver_attachment.id
       WHERE zerver_attachment_messages.message_id IN {message_ids}
            AND zerver_archivedattachment.id IS NULL
       GROUP BY zerver_attachment.id
    """
    move_rows(Attachment, query, message_ids=ids_list_to_sql_query_format(msg_ids))

def move_attachments_message_rows_to_archive(msg_ids: List[int]) -> None:
    if not msg_ids:
        return

    query = """
       INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id, archivedmessage_id)
       SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
           zerver_attachment_messages.message_id
       FROM zerver_attachment_messages
       INNER JOIN zerver_attachment
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       LEFT JOIN zerver_archivedattachment_messages
           ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
       WHERE  zerver_attachment_messages.message_id IN {message_ids}
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    with connection.cursor() as cursor:
        cursor.execute(query.format(message_ids=ids_list_to_sql_query_format(msg_ids)))

def delete_messages(msg_ids: List[int]) -> None:
    Message.objects.filter(id__in=msg_ids).delete()

def delete_expired_attachments() -> None:
    Attachment.objects.filter(
        messages__isnull=True, id__in=ArchivedAttachment.objects.all(),
    ).delete()

def move_related_objects_to_archive(msg_ids: List[int]) -> None:
    move_models_with_message_key_to_archive(msg_ids)
    move_attachments_to_archive(msg_ids)
    move_attachments_message_rows_to_archive(msg_ids)

def archive_messages_by_recipient(recipient: Recipient, message_retention_days: int) -> None:
    msg_ids = move_expired_messages_to_archive_by_recipient(recipient, message_retention_days)
    move_related_objects_to_archive(msg_ids)
    delete_messages(msg_ids)

def archive_personal_and_huddle_messages() -> None:
    for realm in Realm.objects.filter(message_retention_days__isnull=False):
        msg_ids = move_expired_personal_and_huddle_messages_to_archive(realm)
        move_related_objects_to_archive(msg_ids)
        delete_messages(msg_ids)

def archive_stream_messages() -> None:
    streams = Stream.objects.filter(message_retention_days__isnull=False)
    retention_policy_dict = {}  # type: Dict[int, int]
    for stream in streams:
        retention_policy_dict[stream.id] = stream.message_retention_days

    recipients = get_stream_recipients([stream.id for stream in streams])
    for recipient in recipients:
        archive_messages_by_recipient(recipient, retention_policy_dict[recipient.type_id])

def archive_messages() -> None:
    archive_stream_messages()
    archive_personal_and_huddle_messages()

    # Since figuring out which attachments can be deleted requires scanning the whole
    # ArchivedAttachment table, we should do this just once, at the end of the archiving process:
    delete_expired_attachments()

def move_attachment_messages_to_archive_by_message(message_ids: List[int]) -> None:
    # Move attachments messages relation table data to archive.
    id_list = ', '.join(str(message_id) for message_id in message_ids)

    query = """
        INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id,
            archivedmessage_id)
        SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
            zerver_attachment_messages.message_id
        FROM zerver_attachment_messages
        LEFT JOIN zerver_archivedattachment_messages
            ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
        WHERE zerver_attachment_messages.message_id in ({message_ids})
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    with connection.cursor() as cursor:
        cursor.execute(query.format(message_ids=id_list))

@transaction.atomic
def move_messages_to_archive(message_ids: List[int]) -> None:
    messages = list(Message.objects.filter(id__in=message_ids).values())
    if not messages:
        raise Message.DoesNotExist

    ArchivedMessage.objects.bulk_create([ArchivedMessage(**message) for message in messages])

    # Move user_messages to the archive.
    user_messages = UserMessage.objects.filter(
        message_id__in=message_ids).exclude(id__in=ArchivedUserMessage.objects.all()).values()
    user_messages_ids = [user_message['id'] for user_message in user_messages]
    ArchivedUserMessage.objects.bulk_create(
        [ArchivedUserMessage(**user_message) for user_message in user_messages]
    )

    for model in models_with_message_key:
        elements = model['class'].objects.filter(message_id__in=message_ids).exclude(
            id__in=model['archive_class'].objects.all()
        ).values()

        model['archive_class'].objects.bulk_create(
            [model['archive_class'](**element) for element in elements]
        )

    # Move attachments to archive
    attachments = Attachment.objects.filter(messages__id__in=message_ids).exclude(
        id__in=ArchivedAttachment.objects.all()).distinct().values()
    ArchivedAttachment.objects.bulk_create(
        [ArchivedAttachment(**attachment) for attachment in attachments]
    )

    move_attachment_messages_to_archive_by_message(message_ids)

    # Remove data from main tables
    Message.objects.filter(id__in=message_ids).delete()
    UserMessage.objects.filter(id__in=user_messages_ids, message_id__isnull=True).delete()

    archived_attachments = ArchivedAttachment.objects.filter(messages__id__in=message_ids).distinct()
    Attachment.objects.filter(messages__isnull=True, id__in=archived_attachments).delete()
<|endoftext|>


<commit_before>"""

This module contains all

"""

from flask import request
from app import db
from models.test_result import TestResult
from util.json_helpers import JSON_SUCCESS, JSON_FAILURE
from util.rest.rest_auth import requires_router_token
from . import rest_blueprint
from models.config import TestConfiguration
import json

@rest_blueprint.route("/router/get_config", methods=['POST'])
@requires_router_token()
def get_config():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if 'data' in request.form:
        data = json.loads(request.form['data'])
        rec.interface_stats = data

    if not rec:
        return JSON_FAILURE()

    rec.state = 'running'
    db.session.commit()
    rec.save()

    config = TestConfiguration()

    return JSON_SUCCESS(
            config=config.get_config()
        )

@rest_blueprint.route("/router/edit", methods=['POST'])
@requires_router_token()
def edit():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if not rec:
        return JSON_FAILURE(
                reason="Invalid token or IP."
                )
    if 'data' not in request.form:
        return JSON_FAILURE(
                reason="Missing 'data' parameter"
                )
    # Columns allowed to be updated and their types
    columns = TestResult.get_public_columns()

    data = json.loads(request.form['data'])

    for col in columns:
        if col in data:
            col_type = columns[col]
            datum = col_type(data[col])
            setattr(rec, col, datum)

    # Calculate packet loss from the down/up latencies. (-1 = packet lost)
    ploss_cnt = rec.download_latencies.count(-1)
    ploss_cnt += rec.upload_latencies.count(-1)
    ploss_len = len(rec.download_latencies)
    ploss_len += len(rec.upload_latencies)
    rec.packet_loss_under_load = float(ploss_cnt) / float(ploss_len)
    rec.save()

    return JSON_SUCCESS()
<commit_msg>Fixed an issue where the posting partial results from router caused an error.<commit_after>"""

This module contains all

"""

from flask import request
from app import db
from models.test_result import TestResult
from util.json_helpers import JSON_SUCCESS, JSON_FAILURE
from util.rest.rest_auth import requires_router_token
from . import rest_blueprint
from models.config import TestConfiguration
import json

@rest_blueprint.route("/router/get_config", methods=['POST'])
@requires_router_token()
def get_config():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if 'data' in request.form:
        data = json.loads(request.form['data'])
        rec.interface_stats = data

    if not rec:
        return JSON_FAILURE()

    rec.state = 'running'
    db.session.commit()
    rec.save()

    config = TestConfiguration()

    return JSON_SUCCESS(
            config=config.get_config()
        )

@rest_blueprint.route("/router/edit", methods=['POST'])
@requires_router_token()
def edit():
    ip = request.remote_addr
    token = request.form['router_token'].strip()

    rec = TestResult.get_result_by_token_ip(token, ip)

    if not rec:
        return JSON_FAILURE(
                reason="Invalid token or IP."
                )
    if 'data' not in request.form:
        return JSON_FAILURE(
                reason="Missing 'data' parameter"
                )
    # Columns allowed to be updated and their types
    columns = TestResult.get_public_columns()

    data = json.loads(request.form['data'])

    for col in columns:
        if col in data:
            col_type = columns[col]
            datum = col_type(data[col])
            setattr(rec, col, datum)

    ploss_cnt, ploss_len = 0, 0

    # Calculate packet loss from the down/up latencies. (-1 = packet lost)
    if rec.download_latencies:
        ploss_cnt += rec.download_latencies.count(-1)
        ploss_len += len(rec.download_latencies)

    if rec.upload_latencies:
        ploss_cnt += rec.upload_latencies.count(-1)
        ploss_len += len(rec.upload_latencies)

    if ploss_len > 0:
        rec.packet_loss_under_load = float(ploss_cnt) / float(ploss_len)

    rec.save()

    return JSON_SUCCESS()
<|endoftext|>


<commit_before>from django.conf.urls import include, url
from django.views.decorators.cache import cache_page
from rest_framework import routers

import elections.api_views
import parties.api_views
import people.api_views
from api.next import views as next_views
from api.v09 import views as v09views
from parties.api_views import PartyViewSet, PartyRegisterList
from uk_results.views import CandidateResultViewSet, ResultSetViewSet

v09_api_router = routers.DefaultRouter()

v09_api_router.register(r"persons", v09views.PersonViewSet, basename="person")
v09_api_router.register(r"organizations", v09views.OrganizationViewSet)
v09_api_router.register(r"posts", v09views.PostViewSet)
v09_api_router.register(r"elections", v09views.ElectionViewSet)
v09_api_router.register(r"party_sets", v09views.PartySetViewSet)
v09_api_router.register(r"post_elections", v09views.PostExtraElectionViewSet)
v09_api_router.register(r"memberships", v09views.MembershipViewSet)
v09_api_router.register(r"logged_actions", v09views.LoggedActionViewSet)
v09_api_router.register(
    r"extra_fields", v09views.ExtraFieldViewSet, basename="extra_fields"
)
v09_api_router.register(r"person_redirects", v09views.PersonRedirectViewSet)

v09_api_router.register(r"candidate_results", CandidateResultViewSet)
v09_api_router.register(r"result_sets", ResultSetViewSet)

v09_api_router.register(
    r"candidates_for_postcode",
    v09views.CandidatesAndElectionsForPostcodeViewSet,
    basename="candidates-for-postcode",
)

# "Next" is the label we give to the "bleeding edge" or unstable API
next_api_router = routers.DefaultRouter()
next_api_router.register(
    r"people", people.api_views.PersonViewSet, basename="person"
)
next_api_router.register(r"organizations", next_views.OrganizationViewSet)
next_api_router.register(r"elections", elections.api_views.ElectionViewSet)
next_api_router.register(
    r"election_types",
    elections.api_views.ElectionTypesList,
    basename="election_types",
)
next_api_router.register(r"ballots", elections.api_views.BallotViewSet)
next_api_router.register(r"logged_actions", next_views.LoggedActionViewSet)
next_api_router.register(
    r"person_redirects", people.api_views.PersonRedirectViewSet
)

next_api_router.register(r"candidate_results", CandidateResultViewSet)
next_api_router.register(r"result_sets", ResultSetViewSet)
next_api_router.register(r"parties", PartyViewSet)
next_api_router.register(
    r"party_registers", PartyRegisterList, basename="party_register"
)

urlpatterns = [
    # Router views
    url(r"^api/(?P<version>v0.9)/", include(v09_api_router.urls)),
    url(r"^api/(?P<version>next)/", include(next_api_router.urls)),
    # Standard Django views
    url(
        r"^api/current-elections",
        v09views.CurrentElectionsView.as_view(),
        name="current-elections",
    ),
    url(
        r"^post-id-to-party-set.json$",
        cache_page(60 * 60)(v09views.PostIDToPartySetView.as_view()),
        name="post-id-to-party-set",
    ),
    url(
        r"^all-parties.json$",
        cache_page(60 * 60)(parties.api_views.AllPartiesJSONView.as_view()),
        name="all-parties-json-view",
    ),
    url(r"^version.json", v09views.VersionView.as_view(), name="version"),
    url(
        r"^upcoming-elections",
        v09views.UpcomingElectionsView.as_view(),
        name="upcoming-elections",
    ),
]
<commit_msg>Delete results API views<commit_after>from django.conf.urls import include, url
from django.views.decorators.cache import cache_page
from rest_framework import routers

import elections.api_views
import parties.api_views
import people.api_views
from api.next import views as next_views
from api.v09 import views as v09views
from parties.api_views import PartyViewSet, PartyRegisterList
from uk_results.views import CandidateResultViewSet, ResultSetViewSet

v09_api_router = routers.DefaultRouter()

v09_api_router.register(r"persons", v09views.PersonViewSet, basename="person")
v09_api_router.register(r"organizations", v09views.OrganizationViewSet)
v09_api_router.register(r"posts", v09views.PostViewSet)
v09_api_router.register(r"elections", v09views.ElectionViewSet)
v09_api_router.register(r"party_sets", v09views.PartySetViewSet)
v09_api_router.register(r"post_elections", v09views.PostExtraElectionViewSet)
v09_api_router.register(r"memberships", v09views.MembershipViewSet)
v09_api_router.register(r"logged_actions", v09views.LoggedActionViewSet)
v09_api_router.register(
    r"extra_fields", v09views.ExtraFieldViewSet, basename="extra_fields"
)
v09_api_router.register(r"person_redirects", v09views.PersonRedirectViewSet)

v09_api_router.register(r"candidate_results", CandidateResultViewSet)
v09_api_router.register(r"result_sets", ResultSetViewSet)

v09_api_router.register(
    r"candidates_for_postcode",
    v09views.CandidatesAndElectionsForPostcodeViewSet,
    basename="candidates-for-postcode",
)

# "Next" is the label we give to the "bleeding edge" or unstable API
next_api_router = routers.DefaultRouter()
next_api_router.register(
    r"people", people.api_views.PersonViewSet, basename="person"
)
next_api_router.register(r"organizations", next_views.OrganizationViewSet)
next_api_router.register(r"elections", elections.api_views.ElectionViewSet)
next_api_router.register(
    r"election_types",
    elections.api_views.ElectionTypesList,
    basename="election_types",
)
next_api_router.register(r"ballots", elections.api_views.BallotViewSet)
next_api_router.register(r"logged_actions", next_views.LoggedActionViewSet)
next_api_router.register(
    r"person_redirects", people.api_views.PersonRedirectViewSet
)

next_api_router.register(r"parties", PartyViewSet)
next_api_router.register(
    r"party_registers", PartyRegisterList, basename="party_register"
)

urlpatterns = [
    # Router views
    url(r"^api/(?P<version>v0.9)/", include(v09_api_router.urls)),
    url(r"^api/(?P<version>next)/", include(next_api_router.urls)),
    # Standard Django views
    url(
        r"^api/current-elections",
        v09views.CurrentElectionsView.as_view(),
        name="current-elections",
    ),
    url(
        r"^post-id-to-party-set.json$",
        cache_page(60 * 60)(v09views.PostIDToPartySetView.as_view()),
        name="post-id-to-party-set",
    ),
    url(
        r"^all-parties.json$",
        cache_page(60 * 60)(parties.api_views.AllPartiesJSONView.as_view()),
        name="all-parties-json-view",
    ),
    url(r"^version.json", v09views.VersionView.as_view(), name="version"),
    url(
        r"^upcoming-elections",
        v09views.UpcomingElectionsView.as_view(),
        name="upcoming-elections",
    ),
]
<|endoftext|>


<commit_before>#!/usr/bin/python3
import gi
import threading

gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject

GObject.threads_init()
Gst.init(None)

pipeline = Gst.Pipeline()

src = Gst.ElementFactory.make("decklinksrc", None)
src.set_property("connection", 1)
src.set_property("mode", 18)
pipeline.add(src)

aqueue = Gst.ElementFactory.make("queue", None)
pipeline.add(aqueue)

vqueue = Gst.ElementFactory.make("queue", None)
pipeline.add(vqueue)

asink = Gst.ElementFactory.make("autoaudiosink", None)
asink.set_property("sync", False)
pipeline.add(asink)

vsink = Gst.ElementFactory.make("autovideosink", None)
vsink.set_property("sync", False)
pipeline.add(vsink)

src.link(aqueue)
src.link(vqueue)

aqueue.link(asink)
vqueue.link(vsink)

pipeline.set_state(Gst.State.PLAYING)
loop = GObject.MainLoop()
loop.run()
<commit_msg>Add resampler<commit_after>#!/usr/bin/python3
import gi
import threading

gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject

GObject.threads_init()
Gst.init(None)

class AudioResampler(Gst.Bin):
	def __init__(self):
		super(AudioResampler, self).__init__()
		convert = Gst.ElementFactory.make('audioconvert', None)
		resample = Gst.ElementFactory.make('audioresample', None)
		capsfilter = Gst.ElementFactory.make('capsfilter', None)

		capsfilter.set_property('caps', Gst.caps_from_string('audio/x-raw, rate=44100'))

		self.add(convert)
		self.add(resample)
		self.add(capsfilter)

		convert.link(resample)
		resample.link(capsfilter)

		self.add_pad(Gst.GhostPad.new('sink', convert.get_static_pad('sink')))
		self.add_pad(Gst.GhostPad.new('src', capsfilter.get_static_pad('src')))

pipeline = Gst.Pipeline()

src = Gst.ElementFactory.make("decklinksrc", None)
src.set_property("connection", 1)
src.set_property("mode", 18)
pipeline.add(src)

aqueue = Gst.ElementFactory.make("queue", None)
pipeline.add(aqueue)

vqueue = Gst.ElementFactory.make("queue", None)
pipeline.add(vqueue)

ars = AudioResampler()
pipeline.add(ars)

asink = Gst.ElementFactory.make("autoaudiosink", None)
asink.set_property("sync", False)
pipeline.add(asink)

vsink = Gst.ElementFactory.make("autovideosink", None)
vsink.set_property("sync", False)
pipeline.add(vsink)

src.link(aqueue)
src.link(vqueue)

aqueue.link(ars)
ars.link(asink)
vqueue.link(vsink)

pipeline.set_state(Gst.State.PLAYING)
loop = GObject.MainLoop()
loop.run()
<|endoftext|>


<commit_before>import conda_build.conda_interface
import networkx as nx
import conda_build.api
from compute_build_graph import construct_graph
import argparse
import os
from collections import OrderedDict
import sys
import subprocess
import yaml

try:
    from ruamel_yaml import BaseLoader, load
except ImportError:
    from yaml import BaseLoader, load

def get_host_platform():
    from sys import platform
    if platform == "linux" or platform == "linux2":
        return "linux"
    elif platform == "darwin":
        return "osx"
    elif platform == "win32":
        return "win"

def get_config_name(arch):
    platform = get_host_platform()
    return os.environ.get("CONFIG", "{}{}".format(platform, arch))

def build_all(recipes_dir, arch):
    folders = list(filter(lambda d: os.path.isdir(os.path.join(recipes_dir, d)), os.listdir(recipes_dir)))
    if not folders:
        print("Found no recipes to build")
        return

    platform = get_host_platform()
    script_dir = os.path.dirname(os.path.realpath(__file__))
    variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch)))

    found_cuda = False
    found_centos7 = False
    for folder in folders:
        meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml")
        if os.path.exists(meta_yaml):
            with(open(meta_yaml, "r", encoding="utf-8")) as f:
                text = ''.join(f.readlines())
                if 'cuda' in text:
                    found_cuda = True
                if 'sysroot_linux-64' in text:
                    found_centos7 = True
    if found_cuda:
        print('##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1')
    if found_centos7:
        os.environ["DEFAULT_LINUX_VERSION"] = "cos7"

    deployment_version = (0, 0)
    sdk_version = (0, 0)
    channel_urls = None
    for folder in folders:
        cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml")
        if os.path.exists(cbc):
            with open(cbc, "r") as f:
                text = ''.join(f.readlines())
            if platform == 'osx' and (
                    'MACOSX_DEPLOYMENT_TARGET' in text or
                    'MACOSX_SDK_VERSION' in text):
                config = load(text, Loader=BaseLoader)

                if 'MACOSX_DEPLOYMENT_TARGET' in config:
                    for version in config['MACOSX_DEPLOYMENT_TARGET']:
                        version = tuple([int(x) for x in version.split('.')])
                        deployment_version = max(deployment_version, version)
                if 'MACOSX_SDK_VERSION' in config:
                    for version in config['MACOSX_SDK_VERSION']:
                        version = tuple([int(x) for x in version.split('.')])
                        sdk_version = max(sdk_version, deployment_version, version)

            if 'channel_sources' not in text:
                new_channel_urls = ['local', 'conda-forge']
            else:
                config = load(text, Loader=BaseLoader)
                new_channel_urls = ['local'] + config['channel_sources'][0].split(',')
            if channel_urls is None:
                channel_urls = new_channel_urls
            elif channel_urls != new_channel_urls:
                raise ValueError(f'Detected different channel_sources in the recipes: {channel_urls} vs. {new_channel_urls}. Consider submitting them in separate PRs')

    if channel_urls is None:
        channel_urls = ['local', 'conda-forge']

    with open(variant_config_file, 'r') as f:
        variant_text = ''.join(f.readlines())

    if deployment_version != (0, 0):
        deployment_version = '.'.join([str(x) for x in deployment_version])
        print("Overriding MACOSX_DEPLOYMENT_TARGET to be ", deployment_version)
        variant_text += '\nMACOSX_DEPLOYMENT_TARGET:\n'
        variant_text += f'- {deployment_version}\n'

    if sdk_version != (0, 0):
        sdk_version = '.'.join([str(x) for x in sdk_version])
        print("Overriding MACOSX_SDK_VERSION to be ", sdk_version)
        variant_text += '\nMACOSX_SDK_VERSION:\n'
        variant_text += f'- {sdk_version}\n'

    with open(variant_config_file, 'w') as f:
        f.write(variant_text)

    if platform == "osx" and (sdk_version != (0, 0) or deployment_version != (0, 0)):
        subprocess.run("run_conda_forge_build_setup", shell=True, check=True)

    if 'conda-forge' not in channel_urls:
        raise ValueError('conda-forge needs to be part of channel_sources')
    print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
    build_folders(recipes_dir, folders, arch, channel_urls)

def get_config(arch, channel_urls):
    exclusive_config_files = [os.path.join(conda_build.conda_interface.root_dir,
                                           'conda_build_config.yaml')]
    script_dir = os.path.dirname(os.path.realpath(__file__))
    # since variant builds override recipe/conda_build_config.yaml, see
    # https://github.com/conda/conda-build/blob/3.21.8/conda_build/variants.py#L175-L181
    # we need to make sure not to use variant_configs here, otherwise
    # staged-recipes PRs cannot override anything using the recipe-cbc.
    exclusive_config_file = os.path.join(script_dir, '{}.yaml'.format(
        get_config_name(arch)))
    if os.path.exists(exclusive_config_file):
        exclusive_config_files.append(exclusive_config_file)

    error_overlinking = (get_host_platform() != "win")

    config = conda_build.api.Config(
        arch=arch, exclusive_config_files=exclusive_config_files,
        channel_urls=channel_urls, error_overlinking=error_overlinking,
    )
    return config

def build_folders(recipes_dir, folders, arch, channel_urls):

    index_path = os.path.join(sys.exec_prefix, 'conda-bld')
    os.makedirs(index_path, exist_ok=True)
    conda_build.api.update_index(index_path)
    index = conda_build.conda_interface.get_index(channel_urls=channel_urls)
    conda_resolve = conda_build.conda_interface.Resolve(index)

    config = get_config(arch, channel_urls)
    platform = get_host_platform()

    worker = {'platform': platform, 'arch': arch,
              'label': '{}-{}'.format(platform, arch)}

    G = construct_graph(recipes_dir, worker=worker, run='build',
                        conda_resolve=conda_resolve, folders=folders,
                        config=config, finalize=False)
    order = list(nx.topological_sort(G))
    order.reverse()

    print('Computed that there are {} distributions to build from {} recipes'
          .format(len(order), len(folders)))
    if not order:
        print('Nothing to do')
        return
    print("Resolved dependencies, will be built in the following order:")
    print('    '+'\n    '.join(order))

    d = OrderedDict()
    for node in order:
        d[G.nodes[node]['meta'].meta_path] = 1

    for recipe in d.keys():
        conda_build.api.build([recipe], config=get_config(arch, channel_urls))

def check_recipes_in_correct_dir(root_dir, correct_dir):
    from pathlib import Path
    for path in Path(root_dir).rglob('meta.yaml'):
        path = path.absolute().relative_to(root_dir)
        if path.parts[0] == 'build_artifacts':
            # ignore pkg_cache in build_artifacts
            continue
        if path.parts[0] != correct_dir:
            raise RuntimeError(f"recipe {path.parts} in wrong directory")
        if len(path.parts) != 3:
            raise RuntimeError(f"recipe {path.parts} in wrong directory")

def read_mambabuild(recipes_dir):
    folders = os.listdir(recipes_dir)
    use_it = True
    for folder in folders:
        cf = os.path.join(recipes_dir, folder, "conda-forge.yml")
        if os.path.exists(cf):
            with open(cf, "r") as f:
                cfy = yaml.safe_load(f.read())
            use_it = use_it and cfy.get("build_with_mambabuild", True)
    return use_it

def use_mambabuild():
    from boa.cli.mambabuild import prepare
    prepare()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--arch', default='64',
                        help='target architecture (64 or 32)')
    args = parser.parse_args()
    root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    check_recipes_in_correct_dir(root_dir, "recipes")
    use_mamba = read_mambabuild(os.path.join(root_dir, "recipes"))
    if use_mamba:
      use_mambabuild()
      subprocess.run("conda clean --all --yes", shell=True, check=True)
    build_all(os.path.join(root_dir, "recipes"), args.arch)
<commit_msg>BUG put quotes around versions in yaml<commit_after>import conda_build.conda_interface
import networkx as nx
import conda_build.api
from compute_build_graph import construct_graph
import argparse
import os
from collections import OrderedDict
import sys
import subprocess
import yaml

try:
    from ruamel_yaml import BaseLoader, load
except ImportError:
    from yaml import BaseLoader, load

def get_host_platform():
    from sys import platform
    if platform == "linux" or platform == "linux2":
        return "linux"
    elif platform == "darwin":
        return "osx"
    elif platform == "win32":
        return "win"

def get_config_name(arch):
    platform = get_host_platform()
    return os.environ.get("CONFIG", "{}{}".format(platform, arch))

def build_all(recipes_dir, arch):
    folders = list(filter(lambda d: os.path.isdir(os.path.join(recipes_dir, d)), os.listdir(recipes_dir)))
    if not folders:
        print("Found no recipes to build")
        return

    platform = get_host_platform()
    script_dir = os.path.dirname(os.path.realpath(__file__))
    variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch)))

    found_cuda = False
    found_centos7 = False
    for folder in folders:
        meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml")
        if os.path.exists(meta_yaml):
            with(open(meta_yaml, "r", encoding="utf-8")) as f:
                text = ''.join(f.readlines())
                if 'cuda' in text:
                    found_cuda = True
                if 'sysroot_linux-64' in text:
                    found_centos7 = True
    if found_cuda:
        print('##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1')
    if found_centos7:
        os.environ["DEFAULT_LINUX_VERSION"] = "cos7"

    deployment_version = (0, 0)
    sdk_version = (0, 0)
    channel_urls = None
    for folder in folders:
        cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml")
        if os.path.exists(cbc):
            with open(cbc, "r") as f:
                text = ''.join(f.readlines())
            if platform == 'osx' and (
                    'MACOSX_DEPLOYMENT_TARGET' in text or
                    'MACOSX_SDK_VERSION' in text):
                config = load(text, Loader=BaseLoader)

                if 'MACOSX_DEPLOYMENT_TARGET' in config:
                    for version in config['MACOSX_DEPLOYMENT_TARGET']:
                        version = tuple([int(x) for x in version.split('.')])
                        deployment_version = max(deployment_version, version)
                if 'MACOSX_SDK_VERSION' in config:
                    for version in config['MACOSX_SDK_VERSION']:
                        version = tuple([int(x) for x in version.split('.')])
                        sdk_version = max(sdk_version, deployment_version, version)

            if 'channel_sources' not in text:
                new_channel_urls = ['local', 'conda-forge']
            else:
                config = load(text, Loader=BaseLoader)
                new_channel_urls = ['local'] + config['channel_sources'][0].split(',')
            if channel_urls is None:
                channel_urls = new_channel_urls
            elif channel_urls != new_channel_urls:
                raise ValueError(f'Detected different channel_sources in the recipes: {channel_urls} vs. {new_channel_urls}. Consider submitting them in separate PRs')

    if channel_urls is None:
        channel_urls = ['local', 'conda-forge']

    with open(variant_config_file, 'r') as f:
        variant_text = ''.join(f.readlines())

    if deployment_version != (0, 0):
        deployment_version = '.'.join([str(x) for x in deployment_version])
        print("Overriding MACOSX_DEPLOYMENT_TARGET to be ", deployment_version)
        variant_text += '\nMACOSX_DEPLOYMENT_TARGET:\n'
        variant_text += f'- "{deployment_version}"\n'

    if sdk_version != (0, 0):
        sdk_version = '.'.join([str(x) for x in sdk_version])
        print("Overriding MACOSX_SDK_VERSION to be ", sdk_version)
        variant_text += '\nMACOSX_SDK_VERSION:\n'
        variant_text += f'- "{sdk_version}"\n'

    with open(variant_config_file, 'w') as f:
        f.write(variant_text)

    if platform == "osx" and (sdk_version != (0, 0) or deployment_version != (0, 0)):
        subprocess.run("run_conda_forge_build_setup", shell=True, check=True)

    if 'conda-forge' not in channel_urls:
        raise ValueError('conda-forge needs to be part of channel_sources')
    print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
    build_folders(recipes_dir, folders, arch, channel_urls)

def get_config(arch, channel_urls):
    exclusive_config_files = [os.path.join(conda_build.conda_interface.root_dir,
                                           'conda_build_config.yaml')]
    script_dir = os.path.dirname(os.path.realpath(__file__))
    # since variant builds override recipe/conda_build_config.yaml, see
    # https://github.com/conda/conda-build/blob/3.21.8/conda_build/variants.py#L175-L181
    # we need to make sure not to use variant_configs here, otherwise
    # staged-recipes PRs cannot override anything using the recipe-cbc.
    exclusive_config_file = os.path.join(script_dir, '{}.yaml'.format(
        get_config_name(arch)))
    if os.path.exists(exclusive_config_file):
        exclusive_config_files.append(exclusive_config_file)

    error_overlinking = (get_host_platform() != "win")

    config = conda_build.api.Config(
        arch=arch, exclusive_config_files=exclusive_config_files,
        channel_urls=channel_urls, error_overlinking=error_overlinking,
    )
    return config

def build_folders(recipes_dir, folders, arch, channel_urls):

    index_path = os.path.join(sys.exec_prefix, 'conda-bld')
    os.makedirs(index_path, exist_ok=True)
    conda_build.api.update_index(index_path)
    index = conda_build.conda_interface.get_index(channel_urls=channel_urls)
    conda_resolve = conda_build.conda_interface.Resolve(index)

    config = get_config(arch, channel_urls)
    platform = get_host_platform()

    worker = {'platform': platform, 'arch': arch,
              'label': '{}-{}'.format(platform, arch)}

    G = construct_graph(recipes_dir, worker=worker, run='build',
                        conda_resolve=conda_resolve, folders=folders,
                        config=config, finalize=False)
    order = list(nx.topological_sort(G))
    order.reverse()

    print('Computed that there are {} distributions to build from {} recipes'
          .format(len(order), len(folders)))
    if not order:
        print('Nothing to do')
        return
    print("Resolved dependencies, will be built in the following order:")
    print('    '+'\n    '.join(order))

    d = OrderedDict()
    for node in order:
        d[G.nodes[node]['meta'].meta_path] = 1

    for recipe in d.keys():
        conda_build.api.build([recipe], config=get_config(arch, channel_urls))

def check_recipes_in_correct_dir(root_dir, correct_dir):
    from pathlib import Path
    for path in Path(root_dir).rglob('meta.yaml'):
        path = path.absolute().relative_to(root_dir)
        if path.parts[0] == 'build_artifacts':
            # ignore pkg_cache in build_artifacts
            continue
        if path.parts[0] != correct_dir:
            raise RuntimeError(f"recipe {path.parts} in wrong directory")
        if len(path.parts) != 3:
            raise RuntimeError(f"recipe {path.parts} in wrong directory")

def read_mambabuild(recipes_dir):
    folders = os.listdir(recipes_dir)
    use_it = True
    for folder in folders:
        cf = os.path.join(recipes_dir, folder, "conda-forge.yml")
        if os.path.exists(cf):
            with open(cf, "r") as f:
                cfy = yaml.safe_load(f.read())
            use_it = use_it and cfy.get("build_with_mambabuild", True)
    return use_it

def use_mambabuild():
    from boa.cli.mambabuild import prepare
    prepare()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--arch', default='64',
                        help='target architecture (64 or 32)')
    args = parser.parse_args()
    root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    check_recipes_in_correct_dir(root_dir, "recipes")
    use_mamba = read_mambabuild(os.path.join(root_dir, "recipes"))
    if use_mamba:
      use_mambabuild()
      subprocess.run("conda clean --all --yes", shell=True, check=True)
    build_all(os.path.join(root_dir, "recipes"), args.arch)
<|endoftext|>