<commit_before>#!/usr/bin/env python

'''A tiny library for parsing, modifying, and composing SRT files.'''

import functools
import re
from datetime import timedelta
from itertools import groupby

SRT_REGEX = re.compile(
    r'(\d+)\n(\d+:\d+:\d+,\d+) --> (\d+:\d+:\d+,\d+)([^\n]*)\n(.+?)\n\n',
    re.MULTILINE | re.DOTALL,
)

@functools.total_ordering  # pylint: disable=too-few-public-methods
class Subtitle(object):
    r'''
    The metadata relating to a single subtitle. Subtitles are sorted by start
    time by default.

    :param index: The SRT index for this subtitle
    :type index: int
    :param start: The time that the subtitle should start being shown
    :type start: :py:class:`datetime.timedelta`
    :param end: The time that the subtitle should stop being shown
    :type end: :py:class:`datetime.timedelta`
    :param proprietary: Proprietary metadata for this subtitle
    :type proprietary: str
    :param content: The subtitle content
    :type content: str
    '''

    # pylint: disable=too-many-arguments
    def __init__(self, index, start, end, content, proprietary=''):
        self.index = index
        self.start = start
        self.end = end
        self.content = content
        self.proprietary = proprietary

    def __hash__(self):
        return hash(frozenset(self.__dict__.items()))

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __lt__(self, other):
        return self.start < other.start

    def __repr__(self):
        return '<%s:%d>' % (self.__class__.__name__, self.index)

    def to_srt(self):
        r'''
        Convert the current :py:class:`Subtitle` to an SRT block.

        >>> from srt import Subtitle, srt_timestamp_to_timedelta
        >>> sub = Subtitle(
        ...     index=1, start=srt_timestamp_to_timedelta('00:01:02,003'),
        ...     end=srt_timestamp_to_timedelta('00:02:03,004'), content='foo',
        ... )
        >>> sub.to_srt()
        '1\n00:01:02,003 --> 00:02:03,004\nfoo\n\n'

        :returns: The metadata of the current :py:class:`Subtitle` object as an
                  SRT formatted subtitle block
        :rtype: str
        '''
        return '%d\n%s --> %s%s\n%s\n\n' % (
            self.index, timedelta_to_srt_timestamp(self.start),
            timedelta_to_srt_timestamp(self.end), self.proprietary,
            self.content,
        )

def timedelta_to_srt_timestamp(timedelta_timestamp):
    r'''
    Convert a :py:class:`~datetime.timedelta` to an SRT timestamp.

    >>> import srt
    >>> import datetime
    >>> delta = datetime.timedelta(hours=1, minutes=23, seconds=4)
    >>> srt.timedelta_to_srt_timestamp(delta)
    '01:23:04,000'

    :param timedelta_timestamp: The timestamp to convert
    :type timedelta_timestamp: :py:class:`datetime.timedelta`
    :returns: An SRT formatted (HH:MM:SS,mmm) representation of the timestamp
    :rtype: str
    '''
    seconds_in_hour = 3600
    seconds_in_minute = 60

    hrs, secs_remainder = divmod(timedelta_timestamp.seconds, seconds_in_hour)
    mins, secs = divmod(secs_remainder, seconds_in_minute)
    msecs = timedelta_timestamp.microseconds // 1000
    return '%02d:%02d:%02d,%03d' % (hrs, mins, secs, msecs)

def srt_timestamp_to_timedelta(srt_timestamp):
    r'''
    Convert an SRT timestamp to a :py:class:`~datetime.timedelta`.

    >>> import srt
    >>> srt_timestamp = '01:23:04,000'
    >>> srt.srt_timestamp_to_timedelta(srt_timestamp)
    datetime.timedelta(0, 4984)

    :param srt_timestamp: A timestamp in SRT format (HH:MM:SS,mmm)
    :type srt_timestamp: str
    :returns: A timedelta object representing the same timestamp passed in
    :rtype: :py:class:`datetime.timedelta`
    '''
    hrs, mins, secs, msecs = (int(x) for x in re.split('[,:]', srt_timestamp))
    return timedelta(hours=hrs, minutes=mins, seconds=secs, milliseconds=msecs)

def sort_and_reindex(subtitles, start_index=1):
    '''
    Reorder subtitles to be sorted by start time order, and rewrite the indexes
    to be in that same order. This ensures that the SRT file will play in an
    expected fashion after, for example, times were changed in some subtitles
    and they may need to be resorted.

    :param subtitles: :py:class:`Subtitle` objects in any order
    :type subtitles: :term:`iterator` of :py:class:`Subtitle` objects
    :param start_index: the lowest index to use
    :type start_index: int
    '''
    skipped_subs = 0
    for new_index, subtitle in enumerate(sorted(subtitles), start=start_index):
        if not subtitle.content.strip():
            # Drop contentless subtitles, as they don't serve any purpose and
            # might confuse the media player's parser
            skipped_subs += 1
            continue

        subtitle.index = new_index - skipped_subs
        yield subtitle

def parse(srt):
    r'''
    Convert an SRT formatted string to a :term:`generator` of Subtitle objects.

    If you are reading from a file, consider using :py:func:`parse_file`
    instead.

    >>> import srt
    >>> subs = srt.parse("""\
    ... 422
    ... 00:31:39,931 --> 00:31:41,931
    ... Using mainly spoons,
    ...
    ... 423
    ... 00:31:41,933 --> 00:31:43,435
    ... we dig a tunnel under the city and release it into the wild.
    ...
    ... """)
    >>> list(subs)
    [<Subtitle:422>, <Subtitle:423>]

    :param srt: Subtitles in SRT format
    :type srt: str
    :returns: The subtitles contained in the SRT file as py:class:`Subtitle`
              objects
    :rtype: :term:`generator` of :py:class:`Subtitle` objects
    '''
    for match in SRT_REGEX.finditer(srt):
        raw_index, raw_start, raw_end, proprietary, content = match.groups()
        yield Subtitle(
            index=int(raw_index), start=srt_timestamp_to_timedelta(raw_start),
            end=srt_timestamp_to_timedelta(raw_end), content=content,
            proprietary=proprietary,
        )

def parse_file(srt):
    r'''
    Parse an SRT formatted stream into py:func:`Subtitle` objects.

    >>> import srt
    >>> with open('tests/srt_samples/monsters.srt') as srt_f:
    ...     subs = list(srt.parse_file(srt_f))
    ...
    >>> subs
    [<Subtitle:421>, <Subtitle:422>, <Subtitle:423>]

    :param srt: A stream containing SRT formatted data
    :type srt: :py:class:`io.TextIOWrapper`, or something that quacks like one
    :returns: The subtitles contained in the SRT file as Subtitle objects
    :rtype: :term:`generator` of :py:class:`Subtitle` objects
    '''
    srt_chomped = (line.rstrip('\r\n') for line in srt)
    srt_blocks = (
        '\n'.join(srt_lines) + '\n\n'
        for line_has_content, srt_lines in groupby(srt_chomped, bool)
        if line_has_content
    )

    for srt_block in srt_blocks:
        subtitle, = parse(srt_block)
        yield subtitle

def compose(subtitles, reindex=True, start_index=1):
    r'''
    Convert an iterator of :py:class:`Subtitle` objects to a string of joined
    SRT blocks.

    This may be a convienient interface when converting back and forth between
    Python objects and SRT formatted blocks, but you may wish to consider using
    the provided stream interface, :py:func:`compose_file`, instead, which is
    considerably more memory efficient when dealing with particularly large SRT
    data.

    >>> from srt import Subtitle, srt_timestamp_to_timedelta, compose
    >>> subs = [
    ...     Subtitle(
    ...         index=1, start=srt_timestamp_to_timedelta('00:01:02,003'),
    ...         end=srt_timestamp_to_timedelta('00:02:03,004'), content='foo',
    ...     ),
    ...     Subtitle(
    ...         index=2, start=srt_timestamp_to_timedelta('00:03:04,005'),
    ...         end=srt_timestamp_to_timedelta('00:06:07,008'), content='bar',
    ...     ),
    ... ]
    >>> compose(subs)  # doctest: +ELLIPSIS
    '1\n00:01:02,003 --> 00:02:03,004\nfoo\n\n2\n...'

    :param subtitles: The subtitles to convert to SRT blocks
    :type subtitles: :term:`iterator` of :py:class:`Subtitle` objects
    :param reindex: Whether to reindex subtitles based on start time
    :type reindex: bool
    :param start_index: If reindexing, the index to start reindexing from
    :type start_index: int
    :returns: A single SRT formatted string, with each input
              :py:class:`Subtitle` represented as an SRT block
    :rtype: str
    '''
    if reindex:
        subtitles = sort_and_reindex(subtitles, start_index=start_index)
    return ''.join(subtitle.to_srt() for subtitle in subtitles)

def compose_file(subtitles, output, reindex=True, start_index=1):
    r'''
    Stream a sequence of py:class:`Subtitle` objects into an SRT formatted
    stream.

    >>> import tempfile
    >>> from srt import Subtitle, srt_timestamp_to_timedelta, compose_file
    >>> subs = [
    ...     Subtitle(
    ...         index=1, start=srt_timestamp_to_timedelta('00:01:02,003'),
    ...         end=srt_timestamp_to_timedelta('00:02:03,004'), content='foo',
    ...     ),
    ...     Subtitle(
    ...         index=2, start=srt_timestamp_to_timedelta('00:03:04,005'),
    ...         end=srt_timestamp_to_timedelta('00:06:07,008'), content='bar',
    ...     ),
    ... ]
    >>> with tempfile.NamedTemporaryFile(mode='w', encoding='utf8') as srt_f:
    ...    compose_file(subs, srt_f)
    ...
    2

    :param subtitles: :py:class:`Subtitle` objects in the order they should be
                      written to the stream
    :type subtitles: :term:`iterator` of :py:class:`Subtitle` objects
    :param output: A stream to write the resulting SRT blocks to
    :type output: :py:class:`io.TextIOWrapper`, or something that quacks like
                  one
    :param reindex: Whether to reindex subtitles based on start time
    :type reindex: bool
    :param start_index: If reindexing, the index to start reindexing from
    :type start_index: int
    :returns: The number of subtitles that were written to the stream
    :rtype: int
    '''
    num_written = 0

    if reindex:
        subtitles = sort_and_reindex(subtitles, start_index=start_index)

    for num_written, subtitle in enumerate(subtitles, start=1):
        output.write(subtitle.to_srt())

    return num_written
<commit_msg>Remove import statements that are now unnecessary<commit_after>#!/usr/bin/env python

'''A tiny library for parsing, modifying, and composing SRT files.'''

import functools
import re
from datetime import timedelta
from itertools import groupby

SRT_REGEX = re.compile(
    r'(\d+)\n(\d+:\d+:\d+,\d+) --> (\d+:\d+:\d+,\d+)([^\n]*)\n(.+?)\n\n',
    re.MULTILINE | re.DOTALL,
)

@functools.total_ordering  # pylint: disable=too-few-public-methods
class Subtitle(object):
    r'''
    The metadata relating to a single subtitle. Subtitles are sorted by start
    time by default.

    :param index: The SRT index for this subtitle
    :type index: int
    :param start: The time that the subtitle should start being shown
    :type start: :py:class:`datetime.timedelta`
    :param end: The time that the subtitle should stop being shown
    :type end: :py:class:`datetime.timedelta`
    :param proprietary: Proprietary metadata for this subtitle
    :type proprietary: str
    :param content: The subtitle content
    :type content: str
    '''

    # pylint: disable=too-many-arguments
    def __init__(self, index, start, end, content, proprietary=''):
        self.index = index
        self.start = start
        self.end = end
        self.content = content
        self.proprietary = proprietary

    def __hash__(self):
        return hash(frozenset(self.__dict__.items()))

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __lt__(self, other):
        return self.start < other.start

    def __repr__(self):
        return '<%s:%d>' % (self.__class__.__name__, self.index)

    def to_srt(self):
        r'''
        Convert the current :py:class:`Subtitle` to an SRT block.

        >>> sub = Subtitle(
        ...     index=1, start=srt_timestamp_to_timedelta('00:01:02,003'),
        ...     end=srt_timestamp_to_timedelta('00:02:03,004'), content='foo',
        ... )
        >>> sub.to_srt()
        '1\n00:01:02,003 --> 00:02:03,004\nfoo\n\n'

        :returns: The metadata of the current :py:class:`Subtitle` object as an
                  SRT formatted subtitle block
        :rtype: str
        '''
        return '%d\n%s --> %s%s\n%s\n\n' % (
            self.index, timedelta_to_srt_timestamp(self.start),
            timedelta_to_srt_timestamp(self.end), self.proprietary,
            self.content,
        )

def timedelta_to_srt_timestamp(timedelta_timestamp):
    r'''
    Convert a :py:class:`~datetime.timedelta` to an SRT timestamp.

    >>> delta = datetime.timedelta(hours=1, minutes=23, seconds=4)
    >>> srt.timedelta_to_srt_timestamp(delta)
    '01:23:04,000'

    :param timedelta_timestamp: The timestamp to convert
    :type timedelta_timestamp: :py:class:`datetime.timedelta`
    :returns: An SRT formatted (HH:MM:SS,mmm) representation of the timestamp
    :rtype: str
    '''
    seconds_in_hour = 3600
    seconds_in_minute = 60

    hrs, secs_remainder = divmod(timedelta_timestamp.seconds, seconds_in_hour)
    mins, secs = divmod(secs_remainder, seconds_in_minute)
    msecs = timedelta_timestamp.microseconds // 1000
    return '%02d:%02d:%02d,%03d' % (hrs, mins, secs, msecs)

def srt_timestamp_to_timedelta(srt_timestamp):
    r'''
    Convert an SRT timestamp to a :py:class:`~datetime.timedelta`.

    >>> srt_timestamp = '01:23:04,000'
    >>> srt.srt_timestamp_to_timedelta(srt_timestamp)
    datetime.timedelta(0, 4984)

    :param srt_timestamp: A timestamp in SRT format (HH:MM:SS,mmm)
    :type srt_timestamp: str
    :returns: A timedelta object representing the same timestamp passed in
    :rtype: :py:class:`datetime.timedelta`
    '''
    hrs, mins, secs, msecs = (int(x) for x in re.split('[,:]', srt_timestamp))
    return timedelta(hours=hrs, minutes=mins, seconds=secs, milliseconds=msecs)

def sort_and_reindex(subtitles, start_index=1):
    '''
    Reorder subtitles to be sorted by start time order, and rewrite the indexes
    to be in that same order. This ensures that the SRT file will play in an
    expected fashion after, for example, times were changed in some subtitles
    and they may need to be resorted.

    :param subtitles: :py:class:`Subtitle` objects in any order
    :type subtitles: :term:`iterator` of :py:class:`Subtitle` objects
    :param start_index: the lowest index to use
    :type start_index: int
    '''
    skipped_subs = 0
    for new_index, subtitle in enumerate(sorted(subtitles), start=start_index):
        if not subtitle.content.strip():
            # Drop contentless subtitles, as they don't serve any purpose and
            # might confuse the media player's parser
            skipped_subs += 1
            continue

        subtitle.index = new_index - skipped_subs
        yield subtitle

def parse(srt):
    r'''
    Convert an SRT formatted string to a :term:`generator` of Subtitle objects.

    If you are reading from a file, consider using :py:func:`parse_file`
    instead.

    >>> subs = srt.parse("""\
    ... 422
    ... 00:31:39,931 --> 00:31:41,931
    ... Using mainly spoons,
    ...
    ... 423
    ... 00:31:41,933 --> 00:31:43,435
    ... we dig a tunnel under the city and release it into the wild.
    ...
    ... """)
    >>> list(subs)
    [<Subtitle:422>, <Subtitle:423>]

    :param srt: Subtitles in SRT format
    :type srt: str
    :returns: The subtitles contained in the SRT file as py:class:`Subtitle`
              objects
    :rtype: :term:`generator` of :py:class:`Subtitle` objects
    '''
    for match in SRT_REGEX.finditer(srt):
        raw_index, raw_start, raw_end, proprietary, content = match.groups()
        yield Subtitle(
            index=int(raw_index), start=srt_timestamp_to_timedelta(raw_start),
            end=srt_timestamp_to_timedelta(raw_end), content=content,
            proprietary=proprietary,
        )

def parse_file(srt):
    r'''
    Parse an SRT formatted stream into py:func:`Subtitle` objects.

    >>> with open('tests/srt_samples/monsters.srt') as srt_f:
    ...     subs = list(srt.parse_file(srt_f))
    ...
    >>> subs
    [<Subtitle:421>, <Subtitle:422>, <Subtitle:423>]

    :param srt: A stream containing SRT formatted data
    :type srt: :py:class:`io.TextIOWrapper`, or something that quacks like one
    :returns: The subtitles contained in the SRT file as Subtitle objects
    :rtype: :term:`generator` of :py:class:`Subtitle` objects
    '''
    srt_chomped = (line.rstrip('\r\n') for line in srt)
    srt_blocks = (
        '\n'.join(srt_lines) + '\n\n'
        for line_has_content, srt_lines in groupby(srt_chomped, bool)
        if line_has_content
    )

    for srt_block in srt_blocks:
        subtitle, = parse(srt_block)
        yield subtitle

def compose(subtitles, reindex=True, start_index=1):
    r'''
    Convert an iterator of :py:class:`Subtitle` objects to a string of joined
    SRT blocks.

    This may be a convienient interface when converting back and forth between
    Python objects and SRT formatted blocks, but you may wish to consider using
    the provided stream interface, :py:func:`compose_file`, instead, which is
    considerably more memory efficient when dealing with particularly large SRT
    data.

    >>> subs = [
    ...     Subtitle(
    ...         index=1, start=srt_timestamp_to_timedelta('00:01:02,003'),
    ...         end=srt_timestamp_to_timedelta('00:02:03,004'), content='foo',
    ...     ),
    ...     Subtitle(
    ...         index=2, start=srt_timestamp_to_timedelta('00:03:04,005'),
    ...         end=srt_timestamp_to_timedelta('00:06:07,008'), content='bar',
    ...     ),
    ... ]
    >>> compose(subs)  # doctest: +ELLIPSIS
    '1\n00:01:02,003 --> 00:02:03,004\nfoo\n\n2\n...'

    :param subtitles: The subtitles to convert to SRT blocks
    :type subtitles: :term:`iterator` of :py:class:`Subtitle` objects
    :param reindex: Whether to reindex subtitles based on start time
    :type reindex: bool
    :param start_index: If reindexing, the index to start reindexing from
    :type start_index: int
    :returns: A single SRT formatted string, with each input
              :py:class:`Subtitle` represented as an SRT block
    :rtype: str
    '''
    if reindex:
        subtitles = sort_and_reindex(subtitles, start_index=start_index)
    return ''.join(subtitle.to_srt() for subtitle in subtitles)

def compose_file(subtitles, output, reindex=True, start_index=1):
    r'''
    Stream a sequence of py:class:`Subtitle` objects into an SRT formatted
    stream.

    >>> subs = [
    ...     Subtitle(
    ...         index=1, start=srt_timestamp_to_timedelta('00:01:02,003'),
    ...         end=srt_timestamp_to_timedelta('00:02:03,004'), content='foo',
    ...     ),
    ...     Subtitle(
    ...         index=2, start=srt_timestamp_to_timedelta('00:03:04,005'),
    ...         end=srt_timestamp_to_timedelta('00:06:07,008'), content='bar',
    ...     ),
    ... ]
    >>> with tempfile.NamedTemporaryFile(mode='w', encoding='utf8') as srt_f:
    ...    compose_file(subs, srt_f)
    ...
    2

    :param subtitles: :py:class:`Subtitle` objects in the order they should be
                      written to the stream
    :type subtitles: :term:`iterator` of :py:class:`Subtitle` objects
    :param output: A stream to write the resulting SRT blocks to
    :type output: :py:class:`io.TextIOWrapper`, or something that quacks like
                  one
    :param reindex: Whether to reindex subtitles based on start time
    :type reindex: bool
    :param start_index: If reindexing, the index to start reindexing from
    :type start_index: int
    :returns: The number of subtitles that were written to the stream
    :rtype: int
    '''
    num_written = 0

    if reindex:
        subtitles = sort_and_reindex(subtitles, start_index=start_index)

    for num_written, subtitle in enumerate(subtitles, start=1):
        output.write(subtitle.to_srt())

    return num_written
<|endoftext|>


<commit_before>import json
import requests
from bs4 import BeautifulSoup
import xmltodict, json

def parseCityList(citys, pred_date):
    preds = {}
    props = ['pred_temp', 'pred_rain', 'pred_status']
    for i in range(1, len(citys)):
        tds = citys[i].find_all('td')
        city = tds[0].text
        preds[city] = {}
        preds[city]['pred_temp'] = tds[1].text
        preds[city]['pred_rain'] = tds[2].text
        preds[city]['pred_status'] = (tds[3].find('img').get('alt'))
        preds[city]['pred_date'] = pred_date

    return preds

city_preds = {}

req = requests.get('http://www.cwb.gov.tw/V7/forecast/f_index.htm')
req.encoding = 'big-5'
boca_soup = BeautifulSoup(req.text, "html.parser") #get text content of the website

update_at = boca_soup.find('div', attrs={"class": "modifyedDate"}).text

n_area = boca_soup.find("table", attrs={"class": "N_AreaList"}) #find the data table
citys = n_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

c_area = boca_soup.find("table", attrs={"class": "C_AreaList"}) #find the data table
citys = c_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

s_area = boca_soup.find("table", attrs={"class": "S_AreaList"}) #find the data table
citys = s_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

e_area = boca_soup.find("table", attrs={"class": "E_AreaList"}) #find the data table
citys = e_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

a_area = boca_soup.find("table", attrs={"class": "A_AreaList"}) #find the data table
citys = a_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

req = requests.get('http://opendata.cwb.gov.tw/govdownload?dataid=O-A0003-001&authorizationkey=rdec-key-123-45678-011121314')

def write_json(file_name, content):
    with open(file_name, 'w') as output_file:
        json.dump(content, output_file, indent=4)

o = xmltodict.parse(req.text)['cwbopendata']['location']

data = []
for each in o:
    new = {}
    new['lat'] = each['lat']
    new['lng'] = each['lon']
    new['locationName'] = each['locationName']
    new['stationId'] = each['stationId']
    new['obsTime'] = each['time']['obsTime']
    for e in each["weatherElement"]:
        new[e["elementName"]] = e["elementValue"]['value']

    for p in each["parameter"]:
        new[p["parameterName"]] = p["parameterValue"]
    pred = city_preds[new['CITY']]
    for k in pred:
        new[k] = pred[k]

    data.append(new)

write_json('./data/weather.json', data)
<commit_msg>update weather<commit_after>import json
import requests
from bs4 import BeautifulSoup
import xmltodict, json

def parseCityList(citys, pred_date):
    preds = {}
    props = ['pred_temp', 'pred_rain', 'pred_status']
    for i in range(1, len(citys)):
        tds = citys[i].find_all('td')
        city = tds[0].text
        preds[city] = {}
        preds[city]['pred_temp'] = tds[1].text
        preds[city]['pred_rain'] = tds[2].text
        preds[city]['pred_status'] = (tds[3].find('img').get('alt'))
        preds[city]['pred_date'] = pred_date
        print (preds[city]['pred_status'])

    return preds

city_preds = {}

req = requests.get('http://www.cwb.gov.tw/V7/forecast/f_index.htm')
req.encoding = 'big-5'
boca_soup = BeautifulSoup(req.text, "html.parser") #get text content of the website

update_at = boca_soup.find('div', attrs={"class": "modifyedDate"}).text

n_area = boca_soup.find("table", attrs={"class": "N_AreaList"}) #find the data table
citys = n_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

c_area = boca_soup.find("table", attrs={"class": "C_AreaList"}) #find the data table
citys = c_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

s_area = boca_soup.find("table", attrs={"class": "S_AreaList"}) #find the data table
citys = s_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

e_area = boca_soup.find("table", attrs={"class": "E_AreaList"}) #find the data table
citys = e_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

a_area = boca_soup.find("table", attrs={"class": "A_AreaList"}) #find the data table
citys = a_area.find_all('tr')
city_preds.update( parseCityList(citys, update_at) )

req = requests.get('http://opendata.cwb.gov.tw/govdownload?dataid=O-A0003-001&authorizationkey=rdec-key-123-45678-011121314')

def write_json(file_name, content):
    with open(file_name, 'w') as output_file:
        json.dump(content, output_file, indent=4)

o = xmltodict.parse(req.text)['cwbopendata']['location']

data = []
for each in o:
    new = {}
    new['lat'] = each['lat']
    new['lng'] = each['lon']
    new['locationName'] = each['locationName']
    new['stationId'] = each['stationId']
    new['obsTime'] = each['time']['obsTime']
    for e in each["weatherElement"]:
        new[e["elementName"]] = e["elementValue"]['value']

    for p in each["parameter"]:
        new[p["parameterName"]] = p["parameterValue"]
    pred = city_preds[new['CITY']]
    for k in pred:
        new[k] = pred[k]

    data.append(new)

req = requests.get('http://data.gov.tw/iisi/logaccess/3537?dataUrl=http://opendata.cwb.gov.tw/govdownload?dataid=O-A0001-001&authorizationkey=rdec-key-123-45678-011121314&ndctype=XML&ndcnid=9176')

o = xmltodict.parse(req.text)['cwbopendata']['location']

for each in o:
    new = {}
    new['lat'] = each['lat']
    new['lng'] = each['lon']
    new['locationName'] = each['locationName']
    new['stationId'] = each['stationId']
    new['obsTime'] = each['time']['obsTime']
    for e in each["weatherElement"]:
        new[e["elementName"]] = e["elementValue"]['value']

    for p in each["parameter"]:
        new[p["parameterName"]] = p["parameterValue"]
    pred = city_preds[new['CITY']]
    for k in pred:
        new[k] = pred[k]

    data.append(new)

write_json('./data/weather.json', data)
<|endoftext|>


<commit_before>from __future__ import absolute_import

import os
import datetime
import logging

from .utils import async
from .utils.filesystem import ensure_path_exists

def construct_logger_file_path(prefix, suffix):
    ensure_path_exists('./logs')
    timestamp = datetime.datetime.now().strftime(
        '{prefix}-%Y%m%d-%H%M%S-{suffix}.log'.format(
            prefix=prefix, suffix=suffix,
        ),
    )
    return os.path.join('logs', timestamp)

def get_file_logger(name, filename):
    # create logger with 'spam_application'
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    # create file handler which logs even debug messages
    fh = logging.FileHandler(filename)
    fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logging.ERROR)
    # create formatter and add it to the handlers
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)

    return logger

class InterceptedStreamsMixin(object):
    """
    Mixin class for GethProcess instances that feeds all of the stdout and
    stderr lines into some set of provided callback functions.
    """
    stdout_callbacks = None
    stderr_callbacks = None

    def __init__(self, *args, **kwargs):
        super(InterceptedStreamsMixin, self).__init__(*args, **kwargs)
        self.stdout_callbacks = []
        self.stdout_queue = async.JoinableQueue()

        self.stderr_callbacks = []
        self.stderr_queue = async.JoinableQueue()

    def register_stdout_callback(self, callback_fn):
        self.stdout_callbacks.append(callback_fn)

    def register_stderr_callback(self, callback_fn):
        self.stderr_callbacks.append(callback_fn)

    def produce_stdout_queue(self):
        for line in iter(self.proc.stdout.readline, b''):
            self.stdout_queue.put(line)
            async.sleep(0)

    def produce_stderr_queue(self):
        for line in iter(self.proc.stderr.readline, b''):
            self.stderr_queue.put(line)
            async.sleep(0)

    def consume_stdout_queue(self):
        for line in self.stdout_queue:
            for fn in self.stdout_callbacks:
                fn(line.strip())
            self.stdout_queue.task_done()
            async.sleep(0)

    def consume_stderr_queue(self):
        for line in self.stderr_queue:
            for fn in self.stderr_callbacks:
                fn(line.strip())
            self.stderr_queue.task_done()
            async.sleep(0)

    def start(self):
        super(InterceptedStreamsMixin, self).start()

        async.spawn(self.produce_stdout_queue)
        async.spawn(self.produce_stderr_queue)

        async.spawn(self.consume_stdout_queue)
        async.spawn(self.consume_stderr_queue)

    def stop(self):
        super(InterceptedStreamsMixin, self).stop()

        try:
            self.stdout_queue.put(StopIteration)
            self.stdout_queue.join(5)
        except async.Timeout:
            pass

        try:
            self.stderr_queue.put(StopIteration)
            self.stderr_queue.join(5)
        except async.Timeout:
            pass

class LoggingMixin(InterceptedStreamsMixin):
    def __init__(self, *args, **kwargs):
        stdout_logfile_path = kwargs.pop(
            'stdout_logfile_path',
            construct_logger_file_path('geth', 'stdout'),
        )
        stderr_logfile_path = kwargs.pop(
            'stderr_logfile_path',
            construct_logger_file_path('geth', 'stderr'),
        )

        super(LoggingMixin, self).__init__(*args, **kwargs)

        stdout_logger = get_file_logger('geth-stdout', stdout_logfile_path)
        stderr_logger = get_file_logger('geth-stderr', stderr_logfile_path)

        self.register_stdout_callback(stdout_logger.info)
        self.register_stderr_callback(stderr_logger.info)
<commit_msg>timeout on join<commit_after>from __future__ import absolute_import

import os
import datetime
import logging

from .utils import async
from .utils.filesystem import ensure_path_exists

def construct_logger_file_path(prefix, suffix):
    ensure_path_exists('./logs')
    timestamp = datetime.datetime.now().strftime(
        '{prefix}-%Y%m%d-%H%M%S-{suffix}.log'.format(
            prefix=prefix, suffix=suffix,
        ),
    )
    return os.path.join('logs', timestamp)

def get_file_logger(name, filename):
    # create logger with 'spam_application'
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    # create file handler which logs even debug messages
    fh = logging.FileHandler(filename)
    fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logging.ERROR)
    # create formatter and add it to the handlers
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)

    return logger

class InterceptedStreamsMixin(object):
    """
    Mixin class for GethProcess instances that feeds all of the stdout and
    stderr lines into some set of provided callback functions.
    """
    stdout_callbacks = None
    stderr_callbacks = None

    def __init__(self, *args, **kwargs):
        super(InterceptedStreamsMixin, self).__init__(*args, **kwargs)
        self.stdout_callbacks = []
        self.stdout_queue = async.JoinableQueue()

        self.stderr_callbacks = []
        self.stderr_queue = async.JoinableQueue()

    def register_stdout_callback(self, callback_fn):
        self.stdout_callbacks.append(callback_fn)

    def register_stderr_callback(self, callback_fn):
        self.stderr_callbacks.append(callback_fn)

    def produce_stdout_queue(self):
        for line in iter(self.proc.stdout.readline, b''):
            self.stdout_queue.put(line)
            async.sleep(0)

    def produce_stderr_queue(self):
        for line in iter(self.proc.stderr.readline, b''):
            self.stderr_queue.put(line)
            async.sleep(0)

    def consume_stdout_queue(self):
        for line in self.stdout_queue:
            for fn in self.stdout_callbacks:
                fn(line.strip())
            self.stdout_queue.task_done()
            async.sleep(0)

    def consume_stderr_queue(self):
        for line in self.stderr_queue:
            for fn in self.stderr_callbacks:
                fn(line.strip())
            self.stderr_queue.task_done()
            async.sleep(0)

    def start(self):
        super(InterceptedStreamsMixin, self).start()

        async.spawn(self.produce_stdout_queue)
        async.spawn(self.produce_stderr_queue)

        async.spawn(self.consume_stdout_queue)
        async.spawn(self.consume_stderr_queue)

    def stop(self):
        super(InterceptedStreamsMixin, self).stop()

        try:
            self.stdout_queue.put(StopIteration)
            # TODO: figure out how to get a timeout on these joins.
            self.stdout_queue.join()
        except async.Timeout:
            pass

        try:
            self.stderr_queue.put(StopIteration)
            # TODO: figure out how to get a timeout on these joins.
            self.stderr_queue.join()
        except async.Timeout:
            pass

class LoggingMixin(InterceptedStreamsMixin):
    def __init__(self, *args, **kwargs):
        stdout_logfile_path = kwargs.pop(
            'stdout_logfile_path',
            construct_logger_file_path('geth', 'stdout'),
        )
        stderr_logfile_path = kwargs.pop(
            'stderr_logfile_path',
            construct_logger_file_path('geth', 'stderr'),
        )

        super(LoggingMixin, self).__init__(*args, **kwargs)

        stdout_logger = get_file_logger('geth-stdout', stdout_logfile_path)
        stderr_logger = get_file_logger('geth-stderr', stderr_logfile_path)

        self.register_stdout_callback(stdout_logger.info)
        self.register_stderr_callback(stderr_logger.info)
<|endoftext|>


<commit_before>__author__ = 'jonathan'

import itertools
import logging
import traceback
import time

from sqlalchemy.util._collections import KeyedTuple

import uuid
from lib.rome.core.utils import get_objects, is_novabase

from lib.rome.core.models import get_model_classname_from_tablename, get_model_class_from_name
from lib.rome.core.rows.rows_experimental import building_tuples as building_tuples_experimental

from lib.rome.core.lazy_reference import LazyRows

file_logger_enabled = False
try:
    file_logger = logging.getLogger('rome_file_logger')
    hdlr = logging.FileHandler('/opt/logs/rome.log')
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    file_logger.addHandler(hdlr)
    file_logger.setLevel(logging.INFO)
    file_logger_enabled = True
except:
    pass

def all_selectable_are_functions(models):
    return all(x._is_function for x in [y for y in models if not y.is_hidden])

def has_attribute(obj, key):
    if type(obj) is dict:
        return key in obj
    else:
        return hasattr(obj, key)

def set_attribute(obj, key, value):
    if type(obj) is dict:
        obj[key] = value
    else:
        return setattr(obj, key, value)

def get_attribute(obj, key, default=None):
    if type(obj) is dict:
        return obj[key] if key in obj else default
    else:
        return getattr(obj, key, default)

def find_table_name(model):

    """This function return the name of the given model as a String. If the
    model cannot be identified, it returns "none".
    :param model: a model object candidate
    :return: the table name or "none" if the object cannot be identified
    """

    if has_attribute(model, "__tablename__"):
        return model.__tablename__

    if has_attribute(model, "table"):
        return model.table.name

    if has_attribute(model, "class_"):
        return model.class_.__tablename__

    if has_attribute(model, "clauses"):
        for clause in model.clauses:
            return find_table_name(clause)

    return "none"

def extract_models(l):
    already_processed = set()
    result = []
    for selectable in [x for x in l if not x._is_function]:
        if not selectable._model in already_processed:
            already_processed.add(selectable._model)
            result += [selectable]
    return result

def extract_sub_row(row, selectables):

    """Adapt a row result to the expectation of sqlalchemy.
    :param row: a list of python objects
    :param selectables: a list entity class
    :return: the response follows what is required by sqlalchemy (if len(model)==1, a single object is fine, in
    the other case, a KeyTuple where each sub object is associated with it's entity name
    """

    if len(selectables) > 1:

        labels = []

        for selectable in selectables:
            labels += [find_table_name(selectable._model)]

        product = []
        for label in labels:
            product = product + [get_attribute(row, label)]

        # Updating Foreign Keys of objects that are in the row
        # for label in labels:
        #     current_object = get_attribute(row, label)
        #     metadata = current_object.metadata
        #     if metadata and has_attribute(metadata, "_fk_memos"):
        #         for fk_name in metadata._fk_memos:
        #             fks = metadata._fk_memos[fk_name]
        #             for fk in fks:
        #                 local_field_name = fk.column._label
        #                 remote_table_name = fk._colspec.split(".")[-2]
        #                 remote_field_name = fk._colspec.split(".")[-1]
        #
        #                 try:
        #                     remote_object = get_attribute(row, remote_table_name)
        #                     remote_field_value = get_attribute(remote_object, remote_field_name)
        #                     set_attribute(current_object, local_field_name, remote_field_value)
        #                 except:
        #                     pass
        #
        # # Updating fields that are setted to None and that have default values
        # for label in labels:
        #     current_object = get_attribute(row, label)
        #     for field in current_object._sa_class_manager:
        #         instance_state = current_object._sa_instance_state
        #         field_value = get_attribute(current_object, field)
        #         if field_value is None:
        #             try:
        #                 field_column = instance_state.mapper._props[field].columns[0]
        #                 field_default_value = field_column.default.arg
        #                 set_attribute(current_object, field, field_default_value)
        #             except:
        #                 pass

        return KeyedTuple(product, labels=labels)
    else:
        model_name = find_table_name(selectables[0]._model)
        return get_attribute(row, model_name)

def building_tuples(list_results, labels, criterions):
    mode = "not_cartesian_product"
    if mode is "cartesian_product":
        cartesian_product = []
        for element in itertools.product(*list_results):
            cartesian_product += [element]
        return cartesian_product
    else:
        # construct dicts that will keep a ref on objects according to their "id" and "uuid" fields.
        indexed_results = {}
        for i in zip(list_results, labels):
            (results, label) = i
            dict_result = {"id": {}, "uuid": {}}
            for j in results:
                if has_attribute(j, "id"):
                    dict_result["id"][get_attribute(j, "id")] = j
                if has_attribute(j, "uuid"):
                    dict_result["uuid"][get_attribute(j, "uuid")] = j
            indexed_results[label] = dict_result
        # find iteratively pairs that matches according to relationship modelisation
        tuples = []
        tuples_labels = []
        # initialise tuples
        count = 0
        for i in zip(list_results, labels):
            (results, label) = i
            tuples_labels += [label]
            for j in results:
                current_tuple = {label: j}
                tuples += [current_tuple]
            break
        # increase model of exisintg tuples
        count == 0
        for i in zip(list_results, labels):
            if count == 0:
                count += 1
                continue
            (results, label) = i
            tuples_labels += [label]
            # iterate on tuples
            for t in tuples:
                # iterate on existing elements of the current rows
                keys = t.keys()
                for e in keys:
                    model_classname = get_model_classname_from_tablename(e)
                    fake_instance = get_model_class_from_name(model_classname)()
                    relationships = fake_instance.get_relationships()
                    for r in relationships:
                        if r.local_fk_field in ["id", "uuid"]:
                            continue
                        remote_label_name = r.remote_object_tablename
                        if remote_label_name in indexed_results:
                            local_value = get_attribute(t[e], r.local_fk_field)
                            if local_value is not None:
                                try:
                                    remote_candidate = indexed_results[remote_label_name][r.remote_object_field][local_value]
                                    t[remote_label_name] = remote_candidate
                                except Exception as e:
                                    logging.error(e)
                                    traceback.print_exc()
            tuple_groupby_size = {}
            for t in tuples:
                tuple_size = len(t)
                if not tuple_size in tuple_groupby_size:
                    tuple_groupby_size[tuple_size] = []
                tuple_groupby_size[tuple_size] += [t]
            if len(tuple_groupby_size.keys()) > 0:
                max_size = max(tuple_groupby_size.keys())
                tuples = tuple_groupby_size[max_size]
            else:
                tuples = []
        # reordering tuples
        results = []
        for t in tuples:
            if len(t) == len(labels):
                ordered_t = [t[i] for i in labels]
                results += [tuple(ordered_t)]
        return results

def construct_rows(models, criterions, hints):

    """This function constructs the rows that corresponds to the current orm.
    :return: a list of row, according to sqlalchemy expectation
    """

    current_milli_time = lambda: int(round(time.time() * 1000))

    part1_starttime = current_milli_time()

    request_uuid = uuid.uuid1()

    labels = []
    columns = set([])
    rows = []

    model_set = extract_models(models)

    # get the fields of the join result
    for selectable in model_set:
        labels += [find_table_name(selectable._model)]

        if selectable._attributes == "*":
            try:
                selected_attributes = selectable._model._sa_class_manager
            except:
                selected_attributes = selectable._model.class_._sa_class_manager
                pass
        else:
            selected_attributes = [selectable._attributes]

        for field in selected_attributes:

            attribute = None
            if has_attribute(models, "class_"):
                attribute = selectable._model.class_._sa_class_manager[field].__str__()
            elif has_attribute(models, "_sa_class_manager"):
                attribute = selectable._model._sa_class_manager[field].__str__()

            if attribute is not None:
                columns.add(attribute)
    part2_starttime = current_milli_time()

    # loading objects (from database)
    list_results = []
    for selectable in model_set:
        tablename = find_table_name(selectable._model)
        # def filtering_function(n):
        #     print(n.table_name == tablename)
        #     return True
        authorized_secondary_indexes = get_attribute(selectable._model, "_secondary_indexes", [])
        selected_hints = filter(lambda x: x.table_name == tablename and (x.attribute == "id" or x.attribute in authorized_secondary_indexes), hints)
        reduced_hints = map(lambda x:(x.attribute, x.value), selected_hints)
        objects = get_objects(tablename, request_uuid=request_uuid, hints=reduced_hints)
        list_results += [objects]
    part3_starttime = current_milli_time()

    # construct the cartesian product
    tuples = building_tuples(list_results, labels, criterions)
    # tuples = building_tuples_experimental(list_results, labels, criterions)

    part4_starttime = current_milli_time()

    # filtering tuples (cartesian product)
    indexed_rows = {}
    for product in tuples:
        if len(product) > 0:
            row = KeyedTuple(product, labels=labels)
            row_index_key = "%s" % (str(row))

            if row_index_key in indexed_rows:
                continue

            all_criterions_satisfied = True

            for criterion in criterions:
                if not criterion.evaluate(row):
                    all_criterions_satisfied = False
            if all_criterions_satisfied:
                indexed_rows[row_index_key] = True
                rows += [extract_sub_row(row, model_set)]
    part5_starttime = current_milli_time()

    # reordering tuples (+ selecting attributes)
    final_rows = []
    showable_selection = [x for x in models if (not x.is_hidden) or x._is_function]
    part6_starttime = current_milli_time()
    if all_selectable_are_functions(models):
        final_row = []
        for selection in showable_selection:
            value = selection._function._function(rows)
            final_row += [value]
        return [final_row]
    else:
        for row in rows:
            # final_row = []
            final_row = []
            for selection in showable_selection:
                if selection._is_function:
                    value = selection._function._function(rows)
                    final_row += [value]
                else:
                    current_table_name = find_table_name(selection._model)
                    key = current_table_name
                    value = None
                    if not is_novabase(row) and has_attribute(row, key):
                        value = get_attribute(row, key)
                    else:
                        value = row
                    if value is not None:
                        if selection._attributes != "*":
                            final_row += [get_attribute(value, selection._attributes)]
                        else:
                            final_row += [value]
            final_row = LazyRows(final_row)
            if len(showable_selection) == 1:
                final_rows += final_row
            else:
                final_rows += [final_row]
    part7_starttime = current_milli_time()

    query_information = """{"building_query": %s, "loading_objects": %s, "building_tuples": %s, "filtering_tuples": %s, "reordering_columns": %s, "selecting_attributes": %s, "description": "%s", "timestamp": %i}""" % (
        part2_starttime - part1_starttime,
        part3_starttime - part2_starttime,
        part4_starttime - part3_starttime,
        part5_starttime - part4_starttime,
        part6_starttime - part5_starttime,
        part7_starttime - part6_starttime,
        "None",
        current_milli_time()
    )

    logging.info(query_information)
    if file_logger_enabled:
        file_logger.info(query_information)

    return final_rows<commit_msg>trying to merge changes; (5)<commit_after>__author__ = 'jonathan'

import itertools
import logging
import traceback
import time

from sqlalchemy.util._collections import KeyedTuple

import uuid
from lib.rome.core.utils import get_objects, is_novabase

from lib.rome.core.models import get_model_classname_from_tablename, get_model_class_from_name
from lib.rome.core.rows.rows_experimental import building_tuples as building_tuples_experimental

from lib.rome.core.lazy_reference import LazyRows

file_logger_enabled = False
try:
    file_logger = logging.getLogger('rome_file_logger')
    hdlr = logging.FileHandler('/opt/logs/rome.log')
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    file_logger.addHandler(hdlr)
    file_logger.setLevel(logging.INFO)
    file_logger_enabled = True
except:
    pass

def all_selectable_are_functions(models):
    return all(x._is_function for x in [y for y in models if not y.is_hidden])

def has_attribute(obj, key):
    if type(obj) is dict:
        return key in obj
    else:
        return hasattr(obj, key)

def set_attribute(obj, key, value):
    if type(obj) is dict:
        obj[key] = value
    else:
        return setattr(obj, key, value)

def get_attribute(obj, key, default=None):
    if type(obj) is dict:
        return obj[key] if key in obj else default
    else:
        return getattr(obj, key, default)

def find_table_name(model):

    """This function return the name of the given model as a String. If the
    model cannot be identified, it returns "none".
    :param model: a model object candidate
    :return: the table name or "none" if the object cannot be identified
    """

    if has_attribute(model, "__tablename__"):
        return model.__tablename__

    if has_attribute(model, "table"):
        return model.table.name

    if has_attribute(model, "class_"):
        return model.class_.__tablename__

    if has_attribute(model, "clauses"):
        for clause in model.clauses:
            return find_table_name(clause)

    return "none"

def extract_models(l):
    already_processed = set()
    result = []
    for selectable in [x for x in l if not x._is_function]:
        if not selectable._model in already_processed:
            already_processed.add(selectable._model)
            result += [selectable]
    return result

def extract_sub_row(row, selectables):

    """Adapt a row result to the expectation of sqlalchemy.
    :param row: a list of python objects
    :param selectables: a list entity class
    :return: the response follows what is required by sqlalchemy (if len(model)==1, a single object is fine, in
    the other case, a KeyTuple where each sub object is associated with it's entity name
    """

    if len(selectables) > 1:

        labels = []

        for selectable in selectables:
            labels += [find_table_name(selectable._model)]

        product = []
        for label in labels:
            product = product + [get_attribute(row, label)]

        # Updating Foreign Keys of objects that are in the row
        # for label in labels:
        #     current_object = get_attribute(row, label)
        #     metadata = current_object.metadata
        #     if metadata and has_attribute(metadata, "_fk_memos"):
        #         for fk_name in metadata._fk_memos:
        #             fks = metadata._fk_memos[fk_name]
        #             for fk in fks:
        #                 local_field_name = fk.column._label
        #                 remote_table_name = fk._colspec.split(".")[-2]
        #                 remote_field_name = fk._colspec.split(".")[-1]
        #
        #                 try:
        #                     remote_object = get_attribute(row, remote_table_name)
        #                     remote_field_value = get_attribute(remote_object, remote_field_name)
        #                     set_attribute(current_object, local_field_name, remote_field_value)
        #                 except:
        #                     pass
        #
        # # Updating fields that are setted to None and that have default values
        # for label in labels:
        #     current_object = get_attribute(row, label)
        #     for field in current_object._sa_class_manager:
        #         instance_state = current_object._sa_instance_state
        #         field_value = get_attribute(current_object, field)
        #         if field_value is None:
        #             try:
        #                 field_column = instance_state.mapper._props[field].columns[0]
        #                 field_default_value = field_column.default.arg
        #                 set_attribute(current_object, field, field_default_value)
        #             except:
        #                 pass

        return KeyedTuple(product, labels=labels)
    else:
        model_name = find_table_name(selectables[0]._model)
        return get_attribute(row, model_name)

def building_tuples(list_results, labels, criterions):
    mode = "not_cartesian_product"
    if mode is "cartesian_product":
        cartesian_product = []
        for element in itertools.product(*list_results):
            cartesian_product += [element]
        return cartesian_product
    else:
        # construct dicts that will keep a ref on objects according to their "id" and "uuid" fields.
        indexed_results = {}
        for i in zip(list_results, labels):
            (results, label) = i
            dict_result = {"id": {}, "uuid": {}}
            for j in results:
                if has_attribute(j, "id"):
                    dict_result["id"][get_attribute(j, "id")] = j
                if has_attribute(j, "uuid"):
                    dict_result["uuid"][get_attribute(j, "uuid")] = j
            indexed_results[label] = dict_result
        # find iteratively pairs that matches according to relationship modelisation
        tuples = []
        tuples_labels = []
        # initialise tuples
        count = 0
        for i in zip(list_results, labels):
            (results, label) = i
            tuples_labels += [label]
            for j in results:
                current_tuple = {label: j}
                tuples += [current_tuple]
            break
        # increase model of exisintg tuples
        count == 0
        for i in zip(list_results, labels):
            if count == 0:
                count += 1
                continue
            (results, label) = i
            tuples_labels += [label]
            # iterate on tuples
            for t in tuples:
                # iterate on existing elements of the current rows
                keys = t.keys()
                for e in keys:
                    model_classname = get_model_classname_from_tablename(e)
                    fake_instance = get_model_class_from_name(model_classname)()
                    relationships = fake_instance.get_relationships()
                    for r in relationships:
                        if r.local_fk_field in ["id", "uuid"]:
                            continue
                        remote_label_name = r.remote_object_tablename
                        if remote_label_name in indexed_results:
                            local_value = get_attribute(t[e], r.local_fk_field)
                            if local_value is not None:
                                try:
                                    remote_candidate = indexed_results[remote_label_name][r.remote_object_field][local_value]
                                    t[remote_label_name] = remote_candidate
                                except Exception as e:
                                    logging.error(e)
                                    traceback.print_exc()
            tuple_groupby_size = {}
            for t in tuples:
                tuple_size = len(t)
                if not tuple_size in tuple_groupby_size:
                    tuple_groupby_size[tuple_size] = []
                tuple_groupby_size[tuple_size] += [t]
            if len(tuple_groupby_size.keys()) > 0:
                max_size = max(tuple_groupby_size.keys())
                tuples = tuple_groupby_size[max_size]
            else:
                tuples = []
        # reordering tuples
        results = []
        for t in tuples:
            if len(t) == len(labels):
                ordered_t = [t[i] for i in labels]
                results += [tuple(ordered_t)]
        return results

def construct_rows(models, criterions, hints):

    """This function constructs the rows that corresponds to the current orm.
    :return: a list of row, according to sqlalchemy expectation
    """

    current_milli_time = lambda: int(round(time.time() * 1000))

    part1_starttime = current_milli_time()

    request_uuid = uuid.uuid1()

    labels = []
    columns = set([])
    rows = []

    model_set = extract_models(models)

    # get the fields of the join result
    for selectable in model_set:
        labels += [find_table_name(selectable._model)]

        if selectable._attributes == "*":
            try:
                selected_attributes = selectable._model._sa_class_manager
            except:
                selected_attributes = selectable._model.class_._sa_class_manager
                pass
        else:
            selected_attributes = [selectable._attributes]

        for field in selected_attributes:

            attribute = None
            if has_attribute(models, "class_"):
                attribute = selectable._model.class_._sa_class_manager[field].__str__()
            elif has_attribute(models, "_sa_class_manager"):
                attribute = selectable._model._sa_class_manager[field].__str__()

            if attribute is not None:
                columns.add(attribute)
    part2_starttime = current_milli_time()

    # loading objects (from database)
    list_results = []
    for selectable in model_set:
        tablename = find_table_name(selectable._model)
        # def filtering_function(n):
        #     print(n.table_name == tablename)
        #     return True
        authorized_secondary_indexes = get_attribute(selectable._model, "_secondary_indexes", [])
        selected_hints = filter(lambda x: x.table_name == tablename and (x.attribute == "id" or x.attribute in authorized_secondary_indexes), hints)
        reduced_hints = map(lambda x:(x.attribute, x.value), selected_hints)
        objects = get_objects(tablename, request_uuid=request_uuid, hints=reduced_hints)
        list_results += [objects]
    part3_starttime = current_milli_time()

    # construct the cartesian product
    tuples = building_tuples(list_results, labels, criterions)
    # tuples = building_tuples_experimental(list_results, labels, criterions)

    part4_starttime = current_milli_time()

    # filtering tuples (cartesian product)
    indexed_rows = {}
    for product in tuples:
        if len(product) > 0:
            row = KeyedTuple(product, labels=labels)
            row_index_key = "%s" % (str(row))

            if row_index_key in indexed_rows:
                continue

            all_criterions_satisfied = True

            for criterion in criterions:
                if not criterion.evaluate(row):
                    all_criterions_satisfied = False
            if all_criterions_satisfied:
                indexed_rows[row_index_key] = True
                rows += [extract_sub_row(row, model_set)]
    part5_starttime = current_milli_time()

    # reordering tuples (+ selecting attributes)
    final_rows = []
    showable_selection = [x for x in models if (not x.is_hidden) or x._is_function]
    part6_starttime = current_milli_time()
    if all_selectable_are_functions(models):
        final_row = []
        for selection in showable_selection:
            value = selection._function._function(rows)
            final_row += [value]
        return [final_row]
    else:
        for row in rows:
            # final_row = []
            final_row = []
            for selection in showable_selection:
                if selection._is_function:
                    value = selection._function._function(rows)
                    final_row += [value]
                else:
                    current_table_name = find_table_name(selection._model)
                    key = current_table_name
                    value = None
                    if not is_novabase(row) and has_attribute(row, key):
                        value = get_attribute(row, key)
                    else:
                        value = row
                    if value is not None:
                        if selection._attributes != "*":
                            final_row += [get_attribute(value, selection._attributes)]
                        else:
                            final_row += [value]
            # final_row = LazyRows(final_row)
            if len(showable_selection) == 1:
                final_rows += final_row
            else:
                final_rows += [final_row]
    part7_starttime = current_milli_time()

    query_information = """{"building_query": %s, "loading_objects": %s, "building_tuples": %s, "filtering_tuples": %s, "reordering_columns": %s, "selecting_attributes": %s, "description": "%s", "timestamp": %i}""" % (
        part2_starttime - part1_starttime,
        part3_starttime - part2_starttime,
        part4_starttime - part3_starttime,
        part5_starttime - part4_starttime,
        part6_starttime - part5_starttime,
        part7_starttime - part6_starttime,
        "None",
        current_milli_time()
    )

    logging.info(query_information)
    if file_logger_enabled:
        file_logger.info(query_information)

    return final_rows<|endoftext|>


<commit_before>import random
import math

def rounds(right_n, number_recursed):
	user_n = raw_input("Guess: ")
	if right_n == int(user_n):
		print "you are correct"
		return 1
	elif number_recursed == 0:
		print ""
	elif right_n < int(user_n):
		print "too high"
		return rounds(right_n, number_recursed - 1)
	elif right_n > int(user_n):
		print "too low"
		return rounds(right_n, number_recursed - 1)

def round_number(n):
	if n == 0:
		return "You got {} correct".format(str"rounds")
	else:
		right_n = random.randint(0, 100)
		print "You have {} rounds left".format(n)
		round_ = rounds(right_n, 5)
		round_number(n - 1)

def main():
	n = 3
	round_number(n)
main()
<commit_msg>I did it<commit_after>import random
import math

def rounds(right_n, number_recursed, c):
	user_n = raw_input("Guess: ")
	if right_n == int(user_n):
		print """you are correct
"""
		c = c + 1
		return c
	elif number_recursed == 0:
		print ""
	elif right_n < int(user_n):
		print "too high"
		return rounds(right_n, number_recursed - 1, c)
	elif right_n > int(user_n):
		print "too low"
		return rounds(right_n, number_recursed - 1, c)

def round_number(n, c):
	if n == 0:
		print "You got {} correct".format(str(c))
	else:
		right_n = 5
		print "You have {} rounds left".format(n)
		c = rounds(right_n, 5, c)
		round_number(n - 1, c)

def main():
	round_number(3, 0)

main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Written by Daniel Oaks <daniel@danieloaks.net>
# Released under the ISC license
import asyncio

from ircreactor.events import EventManager
from ircreactor.envelope import RFC1459Message

from .capabilities import Capabilities
from .features import Features
from .formatting import unescape
from .info import Info
from .imapping import IDict, IList, IString
from .events import message_to_event
from .utils import validate_hostname

loop = asyncio.get_event_loop()

class ServerConnection(asyncio.Protocol):
    """Manages a connection to a single server."""

    def __init__(self, name=None, reactor=None):
        self.connected = False
        self.ready = False
        self._events_in = EventManager()
        self._events_out = EventManager()
        self._girc_events = EventManager()
        self._new_data = ''

        # we keep a list of imappable entities for us to set the casemap on
        #   when ISUPPORT rolls 'round. we assume the server will keep the same
        #   casemap, so once we've received one we stop keeping track
        self._casemap_set = False
        self._imaps = []

        # name used for this server, eg: rizon
        self.name = name

        # client info
        self.nick = None
        self.user = None
        self.real = None
        self.autojoin_channels = []

        # generated and state info
        self.features = Features(self)  # must be done before info
        self.capabilities = Capabilities(wanted=[
            'account-notify',
            'account-tag',
            'away-notify',
            'cap-notify',
            'chghost',
            'extended-join',
            'invite-notify',
            'metadata',
            'monitor',
            'multi-prefix',
            'sasl',
            'server-time',
            'userhost-in-names',
        ])

        self.info = Info(self)

        # events
        self.register_event('both', 'cap', self.rpl_cap)
        self.register_event('both', 'features', self.rpl_features)
        self.register_event('both', 'endofmotd', self.rpl_endofmotd)
        self.register_event('both', 'nomotd', self.rpl_endofmotd)
        self.register_event('both', 'ping', self.rpl_ping)

        self.reactor = reactor
        self.reactor._append_server(self)

    def set_user_info(self, nick, user='*', real='*'):
        self.nick = nick
        self.user = user
        self.real = real

    # event handling
    def register_event(self, direction, verb, child_fn, priority=10):
        event_managers = []
        if direction in ('in', 'both'):
            event_managers.append(self._events_in)
        if direction in ('out', 'both'):
            event_managers.append(self._events_out)
        if direction == 'girc':
            event_managers.append(self._girc_events)

        for event_manager in event_managers:
            event_manager.register(verb, child_fn, priority=priority)

    # casemapping and casemapped objects
    def set_casemapping(self, casemap):
        if not self._casemap_set:
            self._casemap_set = True

            casemap = casemap.casefold()

            for obj in self._imaps:
                obj.set_std(casemap)

    def istring(self, in_string=''):
        """Return a string that uses this server's IRC casemapping.

        This string's equality with other strings, ``lower()``, and ``upper()`` takes this
        server's casemapping into account. This should be used for things such as nicks and
        channel names, where comparing strings using the correct casemapping can be very
        important.
        """
        new_string = IString(in_string)
        new_string.set_std(self.features.get('casemapping'))
        if not self._casemap_set:
            self._imaps.append(new_string)
        return new_string

    def ilist(self, in_list=[]):
        """Return a list that uses this server's IRC casemapping.

        All strings in this list are lowercased using the server's casemapping before inserting
        them into the list, and the ``in`` operator takes casemapping into account.
        """
        new_list = IList(in_list)
        new_list.set_std(self.features.get('casemapping'))
        if not self._casemap_set:
            self._imaps.append(new_list)
        return new_list

    def idict(self, in_dict={}):
        """Return a dict that uses this server's IRC casemapping.

        All keys in this dictionary are stored and compared using this server's casemapping.
        """
        new_dict = IDict(in_dict)
        new_dict.set_std(self.features.get('casemapping'))
        if not self._casemap_set:
            self._imaps.append(new_dict)
        return new_dict

    # protocol connect / disconnect
    def connection_made(self, transport):
        if not self.nick:
            raise Exception('Nick not found. User info must be set with set_user_info()'
                            'before connecting')
            self.exit()
            return

        peername, port = transport.get_extra_info('peername')
        print('Connected to {}'.format(peername))
        self.transport = transport
        self.connected = True

        self.send('CAP LS', params=['302'])

    def connection_lost(self, exc):
        if not self.connected:
            return
        self.connected = False
        if exc:
            print('Connection error: {}'.format(exc))
            return
        print('Connection closed')

    # protocol send / receive
    def send(self, verb, params=None, source=None, tags=None):
        """Send a generic IRC message to the server.

        A message is created using the various parts of the message, then gets
        assembled and sent to the server.

        Args:
            verb (str): Verb, such as PRIVMSG.
            params (list of str): Message parameters, defaults to no params.
            source (str): Source of the message, defaults to no source.
            tags (dict): `Tags <http://ircv3.net/specs/core/message-tags-3.2.html>`_
                to send with the message.
        """
        m = RFC1459Message.from_data(verb, params=params, source=source, tags=tags)
        self._send_message(m)

    def _send_message(self, message):
        final_message = message.to_message() + '\r\n'

        m = RFC1459Message.from_data('raw')
        m.server = self
        m.data = message.to_message()
        for name, event in message_to_event('out', m):
            self._events_out.dispatch(name, event)

        m = message
        m.server = self
        for name, event in message_to_event('out', m):
            self.info.handle_event_out(event)
            self._events_out.dispatch(name, event)
            self._events_out.dispatch('all', event)

        self.transport.write(bytes(final_message, 'UTF-8'))

    def data_received(self, data):
        # feed in new data from server
        self._new_data += data.decode('UTF-8', 'replace')
        messages = []
        message_buffer = ''

        for char in self._new_data:
            if char in ('\r', '\n'):
                if len(message_buffer):
                    messages.append(message_buffer)
                    message_buffer = ''
                continue

            message_buffer += char

        self._new_data = message_buffer

        # dispatch new messages
        for data in messages:
            m = RFC1459Message.from_data('raw')
            m.server = self
            m.data = data
            for name, event in message_to_event('in', m):
                self._events_in.dispatch(name, event)

            m = RFC1459Message.from_message(data)
            m.server = self
            for name, event in message_to_event('in', m):
                self.info.handle_event_in(event)
                self._events_in.dispatch(name, event)
                self._events_in.dispatch('all', event)

    # commands
    def msg(self, target, message, formatted=True, tags=None):
        """Send a privmsg to the given target."""
        if formatted:
            message = unescape(message)

        self.send('PRIVMSG', params=[target, message], source=self.nick, tags=tags)

    def notice(self, target, message, formatted=True, tags=None):
        """Send a notice to the given target."""
        if formatted:
            message = unescape(message)

        self.send('NOTICE', params=[target, message], source=self.nick, tags=tags)

    def ctcp(self, target, ctcp_verb, argument=None):
        """Send a CTCP request to the given target."""
        # we don't support complex ctcp encapsulation because we're somewhat sane
        atoms = [ctcp_verb]
        if argument is not None:
            atoms.append(argument)
        X_DELIM = '\x01'
        self.msg(target, X_DELIM + ' '.join(atoms) + X_DELIM, formatted=False)

    def ctcp_reply(self, target, ctcp_verb, argument=None):
        """Send a CTCP reply to the given target."""
        # we don't support complex ctcp encapsulation because we're somewhat sane
        atoms = [ctcp_verb]
        if argument is not None:
            atoms.append(argument)
        X_DELIM = '\x01'
        self.notice(target, X_DELIM + ' '.join(atoms) + X_DELIM, formatted=False)

    def join_channel(self, channel, key=None):
        """Join the given channel."""
        params = [channel]
        if key:
            params.append(key)
        self.send('JOIN', params=params)

    def mode(self, target, mode_string=None, tags=None):
        """Sends new modes to or requests existing modes from the given target."""
        params = [target]
        if mode_string:
            params += mode_string
        self.send('MODE', params=params, source=self.nick, tags=tags)

    # default events
    def rpl_cap(self, event):
        if event['direction'] == 'in':
            clientname = event['params'].pop(0)
        subcmd = event['params'].pop(0).casefold()

        self.capabilities.ingest(subcmd, event['params'])

        if self.ready:
            return

        if subcmd == 'ls':
            caps_to_enable = self.capabilities.to_enable
            self.send('CAP', params=['REQ', ' '.join(caps_to_enable)])
        elif subcmd == 'ack':
            self.send('CAP', params=['END'])
            self.send_welcome()
        elif subcmd == 'nak':
            self.send('CAP', params=['END'])
            self.send_welcome()

    def send_welcome(self):
        self.send('NICK', params=[self.nick])
        self.send('USER', params=[self.user, '*', '*', self.real])

    def rpl_features(self, event):
        # last param is 'are supported by this server' text, so we ignore it
        self.features.ingest(*event['params'][:-1])

    def rpl_endofmotd(self, event):
        if not self.ready:
            self.ready = True

            if self.autojoin_channels:
                self.join_channels(*self.autojoin_channels)

    def rpl_ping(self, event):
        self.send('PONG', params=event['params'])

    # convenience
    def is_server(self, name):
        return validate_hostname(name)

    def is_channel(self, name):
        if name[0] in self.features.get('chantypes'):
            return True
        return False

    def is_nick(self, name):
        # XXX - lazy for now, to check ISUPPORT etc
        return not self.is_channel(name) and not self.is_server(name)

    # commands
    def join_channels(self, *channels):
        # we schedule joining for later
        if not self.ready:
            self.autojoin_channels = channels
            return True

        for channel in channels:
            params = []

            if ' ' in channel:
                channel, key = channel.split(' ')
            else:
                key = None

            params.append(channel)
            if key:
                params.append(key)

            self.join_channel(channel, key=key)
<commit_msg>client: Fix IPv6 exception<commit_after>#!/usr/bin/env python3
# Written by Daniel Oaks <daniel@danieloaks.net>
# Released under the ISC license
import asyncio

from ircreactor.events import EventManager
from ircreactor.envelope import RFC1459Message

from .capabilities import Capabilities
from .features import Features
from .formatting import unescape
from .info import Info
from .imapping import IDict, IList, IString
from .events import message_to_event
from .utils import validate_hostname

loop = asyncio.get_event_loop()

class ServerConnection(asyncio.Protocol):
    """Manages a connection to a single server."""

    def __init__(self, name=None, reactor=None):
        self.connected = False
        self.ready = False
        self._events_in = EventManager()
        self._events_out = EventManager()
        self._girc_events = EventManager()
        self._new_data = ''

        # we keep a list of imappable entities for us to set the casemap on
        #   when ISUPPORT rolls 'round. we assume the server will keep the same
        #   casemap, so once we've received one we stop keeping track
        self._casemap_set = False
        self._imaps = []

        # name used for this server, eg: rizon
        self.name = name

        # client info
        self.nick = None
        self.user = None
        self.real = None
        self.autojoin_channels = []

        # generated and state info
        self.features = Features(self)  # must be done before info
        self.capabilities = Capabilities(wanted=[
            'account-notify',
            'account-tag',
            'away-notify',
            'cap-notify',
            'chghost',
            'extended-join',
            'invite-notify',
            'metadata',
            'monitor',
            'multi-prefix',
            'sasl',
            'server-time',
            'userhost-in-names',
        ])

        self.info = Info(self)

        # events
        self.register_event('both', 'cap', self.rpl_cap)
        self.register_event('both', 'features', self.rpl_features)
        self.register_event('both', 'endofmotd', self.rpl_endofmotd)
        self.register_event('both', 'nomotd', self.rpl_endofmotd)
        self.register_event('both', 'ping', self.rpl_ping)

        self.reactor = reactor
        self.reactor._append_server(self)

    def set_user_info(self, nick, user='*', real='*'):
        self.nick = nick
        self.user = user
        self.real = real

    # event handling
    def register_event(self, direction, verb, child_fn, priority=10):
        event_managers = []
        if direction in ('in', 'both'):
            event_managers.append(self._events_in)
        if direction in ('out', 'both'):
            event_managers.append(self._events_out)
        if direction == 'girc':
            event_managers.append(self._girc_events)

        for event_manager in event_managers:
            event_manager.register(verb, child_fn, priority=priority)

    # casemapping and casemapped objects
    def set_casemapping(self, casemap):
        if not self._casemap_set:
            self._casemap_set = True

            casemap = casemap.casefold()

            for obj in self._imaps:
                obj.set_std(casemap)

    def istring(self, in_string=''):
        """Return a string that uses this server's IRC casemapping.

        This string's equality with other strings, ``lower()``, and ``upper()`` takes this
        server's casemapping into account. This should be used for things such as nicks and
        channel names, where comparing strings using the correct casemapping can be very
        important.
        """
        new_string = IString(in_string)
        new_string.set_std(self.features.get('casemapping'))
        if not self._casemap_set:
            self._imaps.append(new_string)
        return new_string

    def ilist(self, in_list=[]):
        """Return a list that uses this server's IRC casemapping.

        All strings in this list are lowercased using the server's casemapping before inserting
        them into the list, and the ``in`` operator takes casemapping into account.
        """
        new_list = IList(in_list)
        new_list.set_std(self.features.get('casemapping'))
        if not self._casemap_set:
            self._imaps.append(new_list)
        return new_list

    def idict(self, in_dict={}):
        """Return a dict that uses this server's IRC casemapping.

        All keys in this dictionary are stored and compared using this server's casemapping.
        """
        new_dict = IDict(in_dict)
        new_dict.set_std(self.features.get('casemapping'))
        if not self._casemap_set:
            self._imaps.append(new_dict)
        return new_dict

    # protocol connect / disconnect
    def connection_made(self, transport):
        if not self.nick:
            raise Exception('Nick not found. User info must be set with set_user_info()'
                            'before connecting')
            self.exit()
            return

        # we do this because IPv6 returns a 4-length tuple
        extra_info = transport.get_extra_info('peername')
        peername, port = extra_info[:2]

        print('Connected to {}'.format(peername))
        self.transport = transport
        self.connected = True

        self.send('CAP LS', params=['302'])

    def connection_lost(self, exc):
        if not self.connected:
            return
        self.connected = False
        if exc:
            print('Connection error: {}'.format(exc))
            return
        print('Connection closed')

    # protocol send / receive
    def send(self, verb, params=None, source=None, tags=None):
        """Send a generic IRC message to the server.

        A message is created using the various parts of the message, then gets
        assembled and sent to the server.

        Args:
            verb (str): Verb, such as PRIVMSG.
            params (list of str): Message parameters, defaults to no params.
            source (str): Source of the message, defaults to no source.
            tags (dict): `Tags <http://ircv3.net/specs/core/message-tags-3.2.html>`_
                to send with the message.
        """
        m = RFC1459Message.from_data(verb, params=params, source=source, tags=tags)
        self._send_message(m)

    def _send_message(self, message):
        final_message = message.to_message() + '\r\n'

        m = RFC1459Message.from_data('raw')
        m.server = self
        m.data = message.to_message()
        for name, event in message_to_event('out', m):
            self._events_out.dispatch(name, event)

        m = message
        m.server = self
        for name, event in message_to_event('out', m):
            self.info.handle_event_out(event)
            self._events_out.dispatch(name, event)
            self._events_out.dispatch('all', event)

        self.transport.write(bytes(final_message, 'UTF-8'))

    def data_received(self, data):
        # feed in new data from server
        self._new_data += data.decode('UTF-8', 'replace')
        messages = []
        message_buffer = ''

        for char in self._new_data:
            if char in ('\r', '\n'):
                if len(message_buffer):
                    messages.append(message_buffer)
                    message_buffer = ''
                continue

            message_buffer += char

        self._new_data = message_buffer

        # dispatch new messages
        for data in messages:
            m = RFC1459Message.from_data('raw')
            m.server = self
            m.data = data
            for name, event in message_to_event('in', m):
                self._events_in.dispatch(name, event)

            m = RFC1459Message.from_message(data)
            m.server = self
            for name, event in message_to_event('in', m):
                self.info.handle_event_in(event)
                self._events_in.dispatch(name, event)
                self._events_in.dispatch('all', event)

    # commands
    def msg(self, target, message, formatted=True, tags=None):
        """Send a privmsg to the given target."""
        if formatted:
            message = unescape(message)

        self.send('PRIVMSG', params=[target, message], source=self.nick, tags=tags)

    def notice(self, target, message, formatted=True, tags=None):
        """Send a notice to the given target."""
        if formatted:
            message = unescape(message)

        self.send('NOTICE', params=[target, message], source=self.nick, tags=tags)

    def ctcp(self, target, ctcp_verb, argument=None):
        """Send a CTCP request to the given target."""
        # we don't support complex ctcp encapsulation because we're somewhat sane
        atoms = [ctcp_verb]
        if argument is not None:
            atoms.append(argument)
        X_DELIM = '\x01'
        self.msg(target, X_DELIM + ' '.join(atoms) + X_DELIM, formatted=False)

    def ctcp_reply(self, target, ctcp_verb, argument=None):
        """Send a CTCP reply to the given target."""
        # we don't support complex ctcp encapsulation because we're somewhat sane
        atoms = [ctcp_verb]
        if argument is not None:
            atoms.append(argument)
        X_DELIM = '\x01'
        self.notice(target, X_DELIM + ' '.join(atoms) + X_DELIM, formatted=False)

    def join_channel(self, channel, key=None):
        """Join the given channel."""
        params = [channel]
        if key:
            params.append(key)
        self.send('JOIN', params=params)

    def mode(self, target, mode_string=None, tags=None):
        """Sends new modes to or requests existing modes from the given target."""
        params = [target]
        if mode_string:
            params += mode_string
        self.send('MODE', params=params, source=self.nick, tags=tags)

    # default events
    def rpl_cap(self, event):
        if event['direction'] == 'in':
            clientname = event['params'].pop(0)
        subcmd = event['params'].pop(0).casefold()

        self.capabilities.ingest(subcmd, event['params'])

        if self.ready:
            return

        if subcmd == 'ls':
            caps_to_enable = self.capabilities.to_enable
            self.send('CAP', params=['REQ', ' '.join(caps_to_enable)])
        elif subcmd == 'ack':
            self.send('CAP', params=['END'])
            self.send_welcome()
        elif subcmd == 'nak':
            self.send('CAP', params=['END'])
            self.send_welcome()

    def send_welcome(self):
        self.send('NICK', params=[self.nick])
        self.send('USER', params=[self.user, '*', '*', self.real])

    def rpl_features(self, event):
        # last param is 'are supported by this server' text, so we ignore it
        self.features.ingest(*event['params'][:-1])

    def rpl_endofmotd(self, event):
        if not self.ready:
            self.ready = True

            if self.autojoin_channels:
                self.join_channels(*self.autojoin_channels)

    def rpl_ping(self, event):
        self.send('PONG', params=event['params'])

    # convenience
    def is_server(self, name):
        return validate_hostname(name)

    def is_channel(self, name):
        if name[0] in self.features.get('chantypes'):
            return True
        return False

    def is_nick(self, name):
        # XXX - lazy for now, to check ISUPPORT etc
        return not self.is_channel(name) and not self.is_server(name)

    # commands
    def join_channels(self, *channels):
        # we schedule joining for later
        if not self.ready:
            self.autojoin_channels = channels
            return True

        for channel in channels:
            params = []

            if ' ' in channel:
                channel, key = channel.split(' ')
            else:
                key = None

            params.append(channel)
            if key:
                params.append(key)

            self.join_channel(channel, key=key)
<|endoftext|>