<commit_before># coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_xpath
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    xpath_text,
)

class AfreecaTVIE(InfoExtractor):
    IE_NAME = 'afreecatv'
    IE_DESC = 'afreecatv.com'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:(?:live|afbbs|www)\.)?afreeca(?:tv)?\.com(?::\d+)?
                            (?:
                                /app/(?:index|read_ucc_bbs)\.cgi|
                                /player/[Pp]layer\.(?:swf|html)
                            )\?.*?\bnTitleNo=|
                            vod\.afreecatv\.com/PLAYER/STATION/
                        )
                        (?P<id>\d+)
                    '''
    _TESTS = [{
        'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=',
        'md5': 'f72c89fe7ecc14c1b5ce506c4996046e',
        'info_dict': {
            'id': '36164052',
            'ext': 'mp4',
            'title': '데일리 에이프릴 요정들의 시상식!',
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
            'upload_date': '20160503',
        },
        'skip': 'Video is gone',
    }, {
        'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867',
        'info_dict': {
            'id': '36153164',
            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
        },
        'playlist_count': 2,
        'playlist': [{
            'md5': 'd8b7c174568da61d774ef0203159bf97',
            'info_dict': {
                'id': '36153164_1',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
                'upload_date': '20160502',
            },
        }, {
            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
            'info_dict': {
                'id': '36153164_2',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
                'upload_date': '20160502',
            },
        }],
        'skip': 'Video is gone',
    }, {
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/18650793',
        'info_dict': {
            'id': '18650793',
            'ext': 'mp4',
            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': '윈아디',
            'uploader_id': 'badkids',
            'duration': 107,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/10481652',
        'info_dict': {
            'id': '10481652',
            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
            'duration': 6492,
        },
        'playlist_count': 2,
        'playlist': [{
            'md5': 'd8b7c174568da61d774ef0203159bf97',
            'info_dict': {
                'id': '20160502_c4c62b9d_174361386_1',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
                'uploader': 'dailyapril',
                'uploader_id': 'dailyapril',
                'upload_date': '20160502',
                'duration': 3601,
            },
        }, {
            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
            'info_dict': {
                'id': '20160502_39e739bb_174361386_2',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
                'uploader': 'dailyapril',
                'uploader_id': 'dailyapril',
                'upload_date': '20160502',
                'duration': 2891,
            },
        }],
        'params': {
            'skip_download': True,
        },
    }, {
        # non standard key
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605',
        'info_dict': {
            'id': '20170411_BE689A0E_190960999_1_2_h',
            'ext': 'mp4',
            'title': '혼자사는여자집',
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': '♥이슬이',
            'uploader_id': 'dasl8121',
            'upload_date': '20170411',
            'duration': 213,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # adult video
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/26542731',
        'info_dict': {
            'id': '20171001_F1AE1711_196617479_1',
            'ext': 'mp4',
            'title': '[생]서아 초심 찾기 방송 (part 1)',
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'BJ서아',
            'uploader_id': 'bjdyrksu',
            'upload_date': '20171001',
            'duration': 3600,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
        'only_matching': True,
    }, {
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/15055030',
        'only_matching': True,
    }]

    @staticmethod
    def parse_video_key(key):
        video_key = {}
        m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
        if m:
            video_key['upload_date'] = m.group('upload_date')
            video_key['part'] = int(m.group('part'))
        return video_key

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        station_id = self._search_regex(
            r'nStationNo\s*=\s*(\d+)', webpage, 'station')
        bbs_id = self._search_regex(
            r'nBbsNo\s*=\s*(\d+)', webpage, 'bbs')
        video_id = self._search_regex(
            r'nTitleNo\s*=\s*(\d+)', webpage, 'title', default=video_id)

        video_xml = self._download_xml(
            'http://afbbs.afreecatv.com:8080/api/video/get_video_info.php',
            video_id, headers={
                'Referer': 'http://vod.afreecatv.com/embed.php',
            }, query={
                'nTitleNo': video_id,
                'nStationNo': station_id,
                'nBbsNo': bbs_id,
                'partialView': 'SKIP_ADULT',
            })

        flag = xpath_text(video_xml, './track/flag', 'flag', default=None)
        if flag and flag != 'SUCCEED':
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, flag), expected=True)

        video_element = video_xml.findall(compat_xpath('./track/video'))[-1]
        if video_element is None or video_element.text is None:
            raise ExtractorError('Specified AfreecaTV video does not exist',
                                 expected=True)

        video_url = video_element.text.strip()

        title = xpath_text(video_xml, './track/title', 'title', fatal=True)

        uploader = xpath_text(video_xml, './track/nickname', 'uploader')
        uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
        duration = int_or_none(xpath_text(
            video_xml, './track/duration', 'duration'))
        thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')

        common_entry = {
            'uploader': uploader,
            'uploader_id': uploader_id,
            'thumbnail': thumbnail,
        }

        info = common_entry.copy()
        info.update({
            'id': video_id,
            'title': title,
            'duration': duration,
        })

        if not video_url:
            entries = []
            file_elements = video_element.findall(compat_xpath('./file'))
            one = len(file_elements) == 1
            for file_num, file_element in enumerate(file_elements, start=1):
                file_url = file_element.text
                if not file_url:
                    continue
                key = file_element.get('key', '')
                upload_date = self._search_regex(
                    r'^(\d{8})_', key, 'upload date', default=None)
                file_duration = int_or_none(file_element.get('duration'))
                format_id = key if key else '%s_%s' % (video_id, file_num)
                if determine_ext(file_url) == 'm3u8':
                    formats = self._extract_m3u8_formats(
                        file_url, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls',
                        note='Downloading part %d m3u8 information' % file_num)
                else:
                    formats = [{
                        'url': file_url,
                        'format_id': 'http',
                    }]
                if not formats:
                    continue
                self._sort_formats(formats)
                file_info = common_entry.copy()
                file_info.update({
                    'id': format_id,
                    'title': title if one else '%s (part %d)' % (title, file_num),
                    'upload_date': upload_date,
                    'duration': file_duration,
                    'formats': formats,
                })
                entries.append(file_info)
            entries_info = info.copy()
            entries_info.update({
                '_type': 'multi_video',
                'entries': entries,
            })
            return entries_info

        info = {
            'id': video_id,
            'title': title,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'duration': duration,
            'thumbnail': thumbnail,
        }

        if determine_ext(video_url) == 'm3u8':
            info['formats'] = self._extract_m3u8_formats(
                video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                m3u8_id='hls')
        else:
            app, playpath = video_url.split('mp4:')
            info.update({
                'url': app,
                'ext': 'flv',
                'play_path': 'mp4:' + playpath,
                'rtmp_live': True,  # downloading won't end without this
            })

        return info
<commit_msg>[afreecatv] Detect deleted videos<commit_after># coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_xpath
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
    xpath_text,
)

class AfreecaTVIE(InfoExtractor):
    IE_NAME = 'afreecatv'
    IE_DESC = 'afreecatv.com'
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:(?:live|afbbs|www)\.)?afreeca(?:tv)?\.com(?::\d+)?
                            (?:
                                /app/(?:index|read_ucc_bbs)\.cgi|
                                /player/[Pp]layer\.(?:swf|html)
                            )\?.*?\bnTitleNo=|
                            vod\.afreecatv\.com/PLAYER/STATION/
                        )
                        (?P<id>\d+)
                    '''
    _TESTS = [{
        'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=',
        'md5': 'f72c89fe7ecc14c1b5ce506c4996046e',
        'info_dict': {
            'id': '36164052',
            'ext': 'mp4',
            'title': '데일리 에이프릴 요정들의 시상식!',
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
            'upload_date': '20160503',
        },
        'skip': 'Video is gone',
    }, {
        'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867',
        'info_dict': {
            'id': '36153164',
            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
        },
        'playlist_count': 2,
        'playlist': [{
            'md5': 'd8b7c174568da61d774ef0203159bf97',
            'info_dict': {
                'id': '36153164_1',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
                'upload_date': '20160502',
            },
        }, {
            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
            'info_dict': {
                'id': '36153164_2',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
                'upload_date': '20160502',
            },
        }],
        'skip': 'Video is gone',
    }, {
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/18650793',
        'info_dict': {
            'id': '18650793',
            'ext': 'mp4',
            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': '윈아디',
            'uploader_id': 'badkids',
            'duration': 107,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/10481652',
        'info_dict': {
            'id': '10481652',
            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'dailyapril',
            'uploader_id': 'dailyapril',
            'duration': 6492,
        },
        'playlist_count': 2,
        'playlist': [{
            'md5': 'd8b7c174568da61d774ef0203159bf97',
            'info_dict': {
                'id': '20160502_c4c62b9d_174361386_1',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
                'uploader': 'dailyapril',
                'uploader_id': 'dailyapril',
                'upload_date': '20160502',
                'duration': 3601,
            },
        }, {
            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
            'info_dict': {
                'id': '20160502_39e739bb_174361386_2',
                'ext': 'mp4',
                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
                'uploader': 'dailyapril',
                'uploader_id': 'dailyapril',
                'upload_date': '20160502',
                'duration': 2891,
            },
        }],
        'params': {
            'skip_download': True,
        },
    }, {
        # non standard key
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605',
        'info_dict': {
            'id': '20170411_BE689A0E_190960999_1_2_h',
            'ext': 'mp4',
            'title': '혼자사는여자집',
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': '♥이슬이',
            'uploader_id': 'dasl8121',
            'upload_date': '20170411',
            'duration': 213,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        # adult video
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/26542731',
        'info_dict': {
            'id': '20171001_F1AE1711_196617479_1',
            'ext': 'mp4',
            'title': '[생]서아 초심 찾기 방송 (part 1)',
            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
            'uploader': 'BJ서아',
            'uploader_id': 'bjdyrksu',
            'upload_date': '20171001',
            'duration': 3600,
            'age_limit': 18,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
        'only_matching': True,
    }, {
        'url': 'http://vod.afreecatv.com/PLAYER/STATION/15055030',
        'only_matching': True,
    }]

    @staticmethod
    def parse_video_key(key):
        video_key = {}
        m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
        if m:
            video_key['upload_date'] = m.group('upload_date')
            video_key['part'] = int(m.group('part'))
        return video_key

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        if re.search(r'alert\(["\']This video has been deleted', webpage):
            raise ExtractorError(
                'Video %s has been deleted' % video_id, expected=True)

        station_id = self._search_regex(
            r'nStationNo\s*=\s*(\d+)', webpage, 'station')
        bbs_id = self._search_regex(
            r'nBbsNo\s*=\s*(\d+)', webpage, 'bbs')
        video_id = self._search_regex(
            r'nTitleNo\s*=\s*(\d+)', webpage, 'title', default=video_id)

        video_xml = self._download_xml(
            'http://afbbs.afreecatv.com:8080/api/video/get_video_info.php',
            video_id, headers={
                'Referer': 'http://vod.afreecatv.com/embed.php',
            }, query={
                'nTitleNo': video_id,
                'nStationNo': station_id,
                'nBbsNo': bbs_id,
                'partialView': 'SKIP_ADULT',
            })

        flag = xpath_text(video_xml, './track/flag', 'flag', default=None)
        if flag and flag != 'SUCCEED':
            raise ExtractorError(
                '%s said: %s' % (self.IE_NAME, flag), expected=True)

        video_element = video_xml.findall(compat_xpath('./track/video'))[-1]
        if video_element is None or video_element.text is None:
            raise ExtractorError(
                'Video %s video does not exist' % video_id, expected=True)

        video_url = video_element.text.strip()

        title = xpath_text(video_xml, './track/title', 'title', fatal=True)

        uploader = xpath_text(video_xml, './track/nickname', 'uploader')
        uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
        duration = int_or_none(xpath_text(
            video_xml, './track/duration', 'duration'))
        thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')

        common_entry = {
            'uploader': uploader,
            'uploader_id': uploader_id,
            'thumbnail': thumbnail,
        }

        info = common_entry.copy()
        info.update({
            'id': video_id,
            'title': title,
            'duration': duration,
        })

        if not video_url:
            entries = []
            file_elements = video_element.findall(compat_xpath('./file'))
            one = len(file_elements) == 1
            for file_num, file_element in enumerate(file_elements, start=1):
                file_url = file_element.text
                if not file_url:
                    continue
                key = file_element.get('key', '')
                upload_date = self._search_regex(
                    r'^(\d{8})_', key, 'upload date', default=None)
                file_duration = int_or_none(file_element.get('duration'))
                format_id = key if key else '%s_%s' % (video_id, file_num)
                if determine_ext(file_url) == 'm3u8':
                    formats = self._extract_m3u8_formats(
                        file_url, video_id, 'mp4', entry_protocol='m3u8_native',
                        m3u8_id='hls',
                        note='Downloading part %d m3u8 information' % file_num)
                else:
                    formats = [{
                        'url': file_url,
                        'format_id': 'http',
                    }]
                if not formats:
                    continue
                self._sort_formats(formats)
                file_info = common_entry.copy()
                file_info.update({
                    'id': format_id,
                    'title': title if one else '%s (part %d)' % (title, file_num),
                    'upload_date': upload_date,
                    'duration': file_duration,
                    'formats': formats,
                })
                entries.append(file_info)
            entries_info = info.copy()
            entries_info.update({
                '_type': 'multi_video',
                'entries': entries,
            })
            return entries_info

        info = {
            'id': video_id,
            'title': title,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'duration': duration,
            'thumbnail': thumbnail,
        }

        if determine_ext(video_url) == 'm3u8':
            info['formats'] = self._extract_m3u8_formats(
                video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                m3u8_id='hls')
        else:
            app, playpath = video_url.split('mp4:')
            info.update({
                'url': app,
                'ext': 'flv',
                'play_path': 'mp4:' + playpath,
                'rtmp_live': True,  # downloading won't end without this
            })

        return info
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
import argparse
import os.path
from zstacklib import *
from datetime import datetime

# create log
logger_dir = "/var/log/zstack/"
create_log(logger_dir)
banner("Starting to deploy image store backup storage agent")
start_time = datetime.now()
# set default value
file_root = "files/imagestorebackupstorage"
pip_url = "https=//pypi.python.org/simple/"
proxy = ""
sproxy = ""
chroot_env = 'false'
zstack_repo = 'false'
current_dir = os.path.dirname(os.path.realpath(__file__))
post_url = ""
fs_rootpath = ""
pkg_imagestorebackupstorage = ""
client = "false"
remote_user = "root"
remote_pass = None
remote_port = None
require_python_env = "false"

# get parameter from shell
parser = argparse.ArgumentParser(description='Deploy image backupstorage to host')
parser.add_argument('-i', type=str, help="""specify inventory host file
                        default=/etc/ansible/hosts""")
parser.add_argument('--private-key', type=str, help='use this file to authenticate the connection')
parser.add_argument('-e', type=str, help='set additional variables as key=value or YAML/JSON')

args = parser.parse_args()
argument_dict = eval(args.e)

# update the variable from shell arguments
locals().update(argument_dict)
imagestore_root = "%s/imagestorebackupstorage/package" % zstack_root

host_post_info = HostPostInfo()
host_post_info.host_inventory = args.i
host_post_info.host = host
host_post_info.post_url = post_url
host_post_info.private_key = args.private_key
host_post_info.remote_user = remote_user
host_post_info.remote_pass = remote_pass
host_post_info.remote_port = remote_port
if remote_pass is not None and remote_user != 'root':
    host_post_info.become = True

# include zstacklib.py
(distro, distro_version, distro_release) = get_remote_host_info(host_post_info)
zstacklib_args = ZstackLibArgs()
zstacklib_args.distro = distro
zstacklib_args.distro_release = distro_release
zstacklib_args.distro_version = distro_version
zstacklib_args.zstack_repo = zstack_repo
zstacklib_args.yum_server = yum_server
zstacklib_args.zstack_root = zstack_root
zstacklib_args.host_post_info = host_post_info
zstacklib_args.pip_url = pip_url
zstacklib_args.trusted_host = trusted_host
zstacklib_args.require_python_env = require_python_env
zstacklib = ZstackLib(zstacklib_args)

if distro == "CentOS" or distro == "RedHat":
    if distro_version < 7:
        qemu_pkg = "qemu-kvm"
    else:
        qemu_pkg = "qemu-kvm-ev-2.3.0"
    if client == "true" :
        if distro_version < 7:
            # change error to warning due to imagestore client will install after add kvm host
            Warning("Imagestore Client only support distribution version newer than 7.0")
        if zstack_repo == 'false':
            yum_install_package(qemu_pkg, host_post_info)
        else:
            command = ("yum clean --enablerepo=alibase metadata && yum --disablerepo=* --enablerepo=%s install "
                       "-y %s") % (zstack_repo, qemu_pkg)
            run_remote_command(command, host_post_info)
    else:
        if zstack_repo == 'false':
            yum_install_package(qemu_pkg, host_post_info)
        else:
            command = ("yum clean --enablerepo=alibase metadata && yum --disablerepo=* --enablerepo=%s install "
                       "-y %s") % (zstack_repo, qemu_pkg)
            run_remote_command(command, host_post_info)

elif distro == "Debian" or distro == "Ubuntu":
    if client == "true" and distro_version < 16:
        Warning("Client only support distribution version newer than 16.04")
    apt_install_packages(["qemu-kvm"], host_post_info)

else:
    error("ERROR: Unsupported distribution")

run_remote_command("rm -rf %s/*" % imagestore_root, host_post_info)
command = 'mkdir -p %s ' % (imagestore_root + "/certs")
run_remote_command(command, host_post_info)

# name: copy imagestore binary
copy_arg = CopyArg()
dest_pkg = "%s/%s" % (imagestore_root, pkg_imagestorebackupstorage)
copy_arg.src = "%s/%s" % (file_root, pkg_imagestorebackupstorage)
copy_arg.dest = dest_pkg
copy(copy_arg, host_post_info)

# name: copy necessary certificates
xs = current_dir.split('/')
local_install_dir = '/'.join(xs[:xs.index('ansible')])
local_cert_dir = os.path.join(local_install_dir, "imagestore", "bin", "certs")

copy_arg = CopyArg()
copy_arg.src = "%s/%s" % (local_cert_dir, "ca.pem")
copy_arg.dest = "%s/%s/%s" % (imagestore_root, "certs", "ca.pem")
copy_arg.args = "mode=644"
copy(copy_arg, host_post_info)

copy_arg = CopyArg()
copy_arg.src = "%s/%s" % (local_cert_dir, "privkey.pem")
copy_arg.dest = "%s/%s/%s" % (imagestore_root, "certs", "privkey.pem")
copy_arg.args = "mode=400"
copy(copy_arg, host_post_info)

# name: install zstack-store
command = "bash %s %s " % (dest_pkg, fs_rootpath)
run_remote_command(command, host_post_info)

# name: restart image store server
if client != "true":
    # integrate zstack-store with init.d
    run_remote_command("/bin/cp -f /usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage /etc/init.d/", host_post_info)
    if distro == "CentOS" or distro == "RedHat":
        command = "/usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage stop && /usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage start && chkconfig zstack-imagestorebackupstorage on"
    elif distro == "Debian" or distro == "Ubuntu":
        command = "/usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage stop && /usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage start && update-rc.d zstack-imagestorebackupstorage enable"
    run_remote_command(command, host_post_info)

host_post_info.start_time = start_time
handle_ansible_info("SUCC: Deploy imagestore backupstore successful", host_post_info, "INFO")
sys.exit(0)
<commit_msg>support qemu-ga installation<commit_after>#!/usr/bin/env python
# encoding: utf-8
import argparse
import os.path
from zstacklib import *
from datetime import datetime

# create log
logger_dir = "/var/log/zstack/"
create_log(logger_dir)
banner("Starting to deploy image store backup storage agent")
start_time = datetime.now()
# set default value
file_root = "files/imagestorebackupstorage"
pip_url = "https=//pypi.python.org/simple/"
proxy = ""
sproxy = ""
chroot_env = 'false'
zstack_repo = 'false'
current_dir = os.path.dirname(os.path.realpath(__file__))
post_url = ""
fs_rootpath = ""
pkg_imagestorebackupstorage = ""
pkg_qemuga = ""
client = "false"
remote_user = "root"
remote_pass = None
remote_port = None
require_python_env = "false"

# get parameter from shell
parser = argparse.ArgumentParser(description='Deploy image backupstorage to host')
parser.add_argument('-i', type=str, help="""specify inventory host file
                        default=/etc/ansible/hosts""")
parser.add_argument('--private-key', type=str, help='use this file to authenticate the connection')
parser.add_argument('-e', type=str, help='set additional variables as key=value or YAML/JSON')

args = parser.parse_args()
argument_dict = eval(args.e)

# update the variable from shell arguments
locals().update(argument_dict)
imagestore_root = "%s/imagestorebackupstorage/package" % zstack_root
qemuga_root = "%s/imagestorebackupstorage/package" % zstack_root

host_post_info = HostPostInfo()
host_post_info.host_inventory = args.i
host_post_info.host = host
host_post_info.post_url = post_url
host_post_info.private_key = args.private_key
host_post_info.remote_user = remote_user
host_post_info.remote_pass = remote_pass
host_post_info.remote_port = remote_port
if remote_pass is not None and remote_user != 'root':
    host_post_info.become = True

# include zstacklib.py
(distro, distro_version, distro_release) = get_remote_host_info(host_post_info)
zstacklib_args = ZstackLibArgs()
zstacklib_args.distro = distro
zstacklib_args.distro_release = distro_release
zstacklib_args.distro_version = distro_version
zstacklib_args.zstack_repo = zstack_repo
zstacklib_args.yum_server = yum_server
zstacklib_args.zstack_root = zstack_root
zstacklib_args.host_post_info = host_post_info
zstacklib_args.pip_url = pip_url
zstacklib_args.trusted_host = trusted_host
zstacklib_args.require_python_env = require_python_env
zstacklib = ZstackLib(zstacklib_args)

if distro == "CentOS" or distro == "RedHat":
    if distro_version < 7:
        qemu_pkg = "qemu-kvm"
    else:
        qemu_pkg = "qemu-kvm-ev-2.3.0"
    if client == "true" :
        if distro_version < 7:
            # change error to warning due to imagestore client will install after add kvm host
            Warning("Imagestore Client only support distribution version newer than 7.0")
        if zstack_repo == 'false':
            yum_install_package(qemu_pkg, host_post_info)
        else:
            command = ("yum clean --enablerepo=alibase metadata && yum --disablerepo=* --enablerepo=%s install "
                       "-y %s") % (zstack_repo, qemu_pkg)
            run_remote_command(command, host_post_info)
    else:
        if zstack_repo == 'false':
            yum_install_package(qemu_pkg, host_post_info)
        else:
            command = ("yum clean --enablerepo=alibase metadata && yum --disablerepo=* --enablerepo=%s install "
                       "-y %s") % (zstack_repo, qemu_pkg)
            run_remote_command(command, host_post_info)

elif distro == "Debian" or distro == "Ubuntu":
    if client == "true" and distro_version < 16:
        Warning("Client only support distribution version newer than 16.04")
    apt_install_packages(["qemu-kvm"], host_post_info)

else:
    error("ERROR: Unsupported distribution")

run_remote_command("rm -rf %s/*" % imagestore_root, host_post_info)
command = 'mkdir -p %s ' % (imagestore_root + "/certs")
run_remote_command(command, host_post_info)

# name: copy qemu-ga binary
copy_arg = CopyArg()
dest_qga_pkg = "%s/%s" % (qemuga_root, pkg_qemuga)
copy_arg.src = "%s/%s" % (file_root, pkg_qemuga)
copy_arg.dest = dest_qga_pkg
copy(copy_arg, host_post_info)

# name: copy imagestore binary
copy_arg = CopyArg()
dest_pkg = "%s/%s" % (imagestore_root, pkg_imagestorebackupstorage)
copy_arg.src = "%s/%s" % (file_root, pkg_imagestorebackupstorage)
copy_arg.dest = dest_pkg
copy(copy_arg, host_post_info)

# name: copy necessary certificates
xs = current_dir.split('/')
local_install_dir = '/'.join(xs[:xs.index('ansible')])
local_cert_dir = os.path.join(local_install_dir, "imagestore", "bin", "certs")

copy_arg = CopyArg()
copy_arg.src = "%s/%s" % (local_cert_dir, "ca.pem")
copy_arg.dest = "%s/%s/%s" % (imagestore_root, "certs", "ca.pem")
copy_arg.args = "mode=644"
copy(copy_arg, host_post_info)

copy_arg = CopyArg()
copy_arg.src = "%s/%s" % (local_cert_dir, "privkey.pem")
copy_arg.dest = "%s/%s/%s" % (imagestore_root, "certs", "privkey.pem")
copy_arg.args = "mode=400"
copy(copy_arg, host_post_info)

# name: install zstack-store
command = "bash %s %s " % (dest_pkg, fs_rootpath)
run_remote_command(command, host_post_info)
# name: install qemu-ga
command = "bash %s %s " % (dest_qga_pkg, pkg_qemuga)
Warning("################ %s" % command)
handle_ansible_info("################"+command, host_post_info, "INFO")
#run_remote_command(command, host_post_info)

# name: restart image store server
if client != "true":
    # integrate zstack-store with init.d
    run_remote_command("/bin/cp -f /usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage /etc/init.d/", host_post_info)
    if distro == "CentOS" or distro == "RedHat":
        command = "/usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage stop && /usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage start && chkconfig zstack-imagestorebackupstorage on"
    elif distro == "Debian" or distro == "Ubuntu":
        command = "/usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage stop && /usr/local/zstack/imagestore/bin/zstack-imagestorebackupstorage start && update-rc.d zstack-imagestorebackupstorage enable"
    run_remote_command(command, host_post_info)

host_post_info.start_time = start_time
handle_ansible_info("SUCC: Deploy imagestore backupstore successful", host_post_info, "INFO")
sys.exit(0)
<|endoftext|>


<commit_before>from django.http.response import Http404, HttpResponseRedirect
import models
import constants

from django.views.generic import DetailView
from django.views.generic.detail import SingleObjectMixin
from django.views.generic.base import View

from core.utils import LoginMixin
from class_based_comments.views import CommentViewGeneric

class VotingDetail(LoginMixin, DetailView, CommentViewGeneric):
    template_name = 'voting_round.html'
    view_name = 'voting'
    model = models.VotingRound
    context_object_name = 'voting_round'

class CastVote(LoginMixin, SingleObjectMixin, View):
    """
    View for casting votes... Be design it is not secure against malicious
    linking.
    """
    model = models.VotingRound

    def get(self, request, **kwargs):
        voting_round = self.get_object()
        action = kwargs['action']
        if action not in constants.VOTES:
            raise Http404()

        vote, created = models.Vote.objects.get_or_create(
            author=request.user,
            round=voting_round,
            defaults={'vote': action})

        if not created and vote.vote != action:
            vote.vote = action
            vote.save()

        return HttpResponseRedirect(voting_round.get_absolute_url())<commit_msg>I like my imports sorted<commit_after>import models
import constants

from django.views.generic import DetailView
from django.views.generic.detail import SingleObjectMixin
from django.views.generic.base import View
from django.http.response import Http404, HttpResponseRedirect

from core.utils import LoginMixin
from class_based_comments.views import CommentViewGeneric

class VotingDetail(LoginMixin, DetailView, CommentViewGeneric):
    template_name = 'voting_round.html'
    view_name = 'voting'
    model = models.VotingRound
    context_object_name = 'voting_round'

class CastVote(LoginMixin, SingleObjectMixin, View):
    """
    View for casting votes... Be design it is not secure against malicious
    linking.
    """
    model = models.VotingRound

    def get(self, request, **kwargs):
        voting_round = self.get_object()
        action = kwargs['action']
        if action not in constants.VOTES:
            raise Http404()

        vote, created = models.Vote.objects.get_or_create(
            author=request.user,
            round=voting_round,
            defaults={'vote': action})

        if not created and vote.vote != action:
            vote.vote = action
            vote.save()

        return HttpResponseRedirect(voting_round.get_absolute_url())<|endoftext|>


<commit_before>import time
import subprocess
try:
    from pipes import quote # Python2 & Python3 <= 3.2
except ImportError:
    from shlex import quote # Python3 >= 3.3

from .stream import async_file_reading

# Prevent to import subprocess only for common classes
CalledProcessError = subprocess.CalledProcessError

# Alternative subprocess methods ---------------------------------------

def check_output(args, stderr=subprocess.STDOUT, shell=True, **kwargs):
    """Run command with arguments and return its output as a byte string

    Overwrite some of the arguments to capture standard error in the result
    and use shell by default before calling subprocess.check_output.

    """
    return subprocess.check_output(args, stderr=stderr, shell=shell, **kwargs)

# Call with stream access ----------------------------------------------

def call_async_output(args, callback, **kwargs):
    """Run command and provide its output asynchronously

    Run command with arguments and wait for it to complete to return the
    returncode attribute. The `callback` can be a method or a 2-tuple of
    methods - for stdout and stderr respectively - which must take one
    byte string argument. It will be called each time the command produces
    some output.

    The stdout and stderr additional arguments for the Popen constructor
    are not allowed as they are used internally.

    Keyword arguments:
        - args -- String or sequence of program arguments
        - callback -- Method or object to call with output as argument
        - **kwargs -- Additional arguments for the Popen constructor

    Returns:
        Exit status of the command

    """
    for a in ['stdout', 'stderr']:
        if a in kwargs:
            raise ValueError('%s argument not allowed, '
                             'it will be overridden.' % a)

    # Validate callback argument
    if isinstance(callback, tuple):
        if len(callback) != 2:
            raise ValueError('callback argument should be a 2-tuple')
        kwargs['stdout'] = kwargs['stderr'] = subprocess.PIPE
        separate_stderr = True
    elif callable(callback):
        kwargs['stdout'] = subprocess.PIPE
        kwargs['stderr'] = subprocess.STDOUT
        separate_stderr = False
        callback = (callback,)
    else:
        raise ValueError('callback argument must be callable or a 2-tuple')

    # Run the command
    p = subprocess.Popen(args, **kwargs)

    # Wrap and get command outputs
    stdout_reader, stdout_consum = async_file_reading(p.stdout, callback[0])
    if separate_stderr:
        stderr_reader, stderr_consum = async_file_reading(p.stderr, callback[1])
        while not stdout_reader.eof() and not stderr_reader.eof():
            time.sleep(.1)
        stderr_reader.join()
        stderr_consum.join()
    else:
        while not stdout_reader.eof():
            time.sleep(.1)
    stdout_reader.join()
    stdout_consum.join()
    return p.poll()

# Call multiple commands -----------------------------------------------

def check_commands(cmds, raise_on_error=False, callback=None,
                   separate_stderr=False, shell=True, **kwargs):
    """Run multiple commands with error management

    Run a list of commands and allow to manage how to treat errors either
    with raise_on_error or callback arguments.

    If callback is provided, it will be called when the command returns
    a non-zero exit code. The callback must take 3 arguments; the
    returncode, the command which failed and the command output. The
    callback should return either False to stop commands execution or True
    to continue.
    Otherwise, if raise_on_error is True a CalledProcessError exception will
    be raised when a command returns a non-zero exit code.

    If callback is provided or raise_on_error is False, all commands will
    be executed and the number of failed commands will be returned.

    The standard output and error of a failed command can be separated with
    separate_stderr set to True. In that case, the output argument passed to
    the callback or the output attribute of the CalledProcessError exception
    will be a 2-tuple containing stdout and stderr as byte strings.

    Keyword arguments:
        - cmds -- List of commands to run
        - raise_on_error -- True to raise a CalledProcessError on error if
            no callback is provided
        - callback -- Method or object to call on command failure
        - separate_stderr -- True to return command output as a 2-tuple
        - **kwargs -- Additional arguments for the Popen constructor

    Returns:
        Number of failed commands

    """
    for a in ['stdout', 'stderr']:
        if a in kwargs:
            raise ValueError('%s argument not allowed, '
                             'it will be overridden.' % a)
    error = 0

    if callback is None:
        if raise_on_error:
            # Raise on command failure
            def callback(r, c, o):
                raise CalledProcessError(r, c, o)
        else:
            # Continue commands execution
            callback = lambda r,c,o: True
    elif not callable(callback):
        raise ValueError('callback argument must be callable')

    # Manage stderr
    if separate_stderr:
        _stderr = subprocess.PIPE
        _get_output = lambda o,e: (o,e)
    else:
        _stderr = subprocess.STDOUT
        _get_output = lambda o,e: o

    # Iterate over commands
    for cmd in cmds:
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                   stderr=_stderr, shell=shell, **kwargs)
        output = _get_output(*process.communicate())
        retcode = process.poll()
        if retcode:
            error += 1
            if not callback(retcode, cmd, output):
                break
    return error
<commit_msg>[fix] edgy bug on slow hardware, especially on our (futur) CI<commit_after>import time
import subprocess
try:
    from pipes import quote # Python2 & Python3 <= 3.2
except ImportError:
    from shlex import quote # Python3 >= 3.3

from .stream import async_file_reading

# Prevent to import subprocess only for common classes
CalledProcessError = subprocess.CalledProcessError

# Alternative subprocess methods ---------------------------------------

def check_output(args, stderr=subprocess.STDOUT, shell=True, **kwargs):
    """Run command with arguments and return its output as a byte string

    Overwrite some of the arguments to capture standard error in the result
    and use shell by default before calling subprocess.check_output.

    """
    return subprocess.check_output(args, stderr=stderr, shell=shell, **kwargs)

# Call with stream access ----------------------------------------------

def call_async_output(args, callback, **kwargs):
    """Run command and provide its output asynchronously

    Run command with arguments and wait for it to complete to return the
    returncode attribute. The `callback` can be a method or a 2-tuple of
    methods - for stdout and stderr respectively - which must take one
    byte string argument. It will be called each time the command produces
    some output.

    The stdout and stderr additional arguments for the Popen constructor
    are not allowed as they are used internally.

    Keyword arguments:
        - args -- String or sequence of program arguments
        - callback -- Method or object to call with output as argument
        - **kwargs -- Additional arguments for the Popen constructor

    Returns:
        Exit status of the command

    """
    for a in ['stdout', 'stderr']:
        if a in kwargs:
            raise ValueError('%s argument not allowed, '
                             'it will be overridden.' % a)

    # Validate callback argument
    if isinstance(callback, tuple):
        if len(callback) != 2:
            raise ValueError('callback argument should be a 2-tuple')
        kwargs['stdout'] = kwargs['stderr'] = subprocess.PIPE
        separate_stderr = True
    elif callable(callback):
        kwargs['stdout'] = subprocess.PIPE
        kwargs['stderr'] = subprocess.STDOUT
        separate_stderr = False
        callback = (callback,)
    else:
        raise ValueError('callback argument must be callable or a 2-tuple')

    # Run the command
    p = subprocess.Popen(args, **kwargs)

    # Wrap and get command outputs
    stdout_reader, stdout_consum = async_file_reading(p.stdout, callback[0])
    if separate_stderr:
        stderr_reader, stderr_consum = async_file_reading(p.stderr, callback[1])
        while not stdout_reader.eof() and not stderr_reader.eof():
            time.sleep(.1)
        stderr_reader.join()
        stderr_consum.join()
    else:
        while not stdout_reader.eof():
            time.sleep(.1)
    stdout_reader.join()
    stdout_consum.join()

    # on slow hardware, in very edgy situations it is possible that the process
    # isn't finished just after having closed stdout and stderr, so we wait a
    # bit to give hime the time to finish (while having a timeout)
    # Note : p.poll() returns None is the process hasn't finished yet
    start = time.time()
    while time.time() - start < 10:
        if p.poll() is not None:
            return p.poll()
        time.sleep(.1)

    return p.poll()

# Call multiple commands -----------------------------------------------

def check_commands(cmds, raise_on_error=False, callback=None,
                   separate_stderr=False, shell=True, **kwargs):
    """Run multiple commands with error management

    Run a list of commands and allow to manage how to treat errors either
    with raise_on_error or callback arguments.

    If callback is provided, it will be called when the command returns
    a non-zero exit code. The callback must take 3 arguments; the
    returncode, the command which failed and the command output. The
    callback should return either False to stop commands execution or True
    to continue.
    Otherwise, if raise_on_error is True a CalledProcessError exception will
    be raised when a command returns a non-zero exit code.

    If callback is provided or raise_on_error is False, all commands will
    be executed and the number of failed commands will be returned.

    The standard output and error of a failed command can be separated with
    separate_stderr set to True. In that case, the output argument passed to
    the callback or the output attribute of the CalledProcessError exception
    will be a 2-tuple containing stdout and stderr as byte strings.

    Keyword arguments:
        - cmds -- List of commands to run
        - raise_on_error -- True to raise a CalledProcessError on error if
            no callback is provided
        - callback -- Method or object to call on command failure
        - separate_stderr -- True to return command output as a 2-tuple
        - **kwargs -- Additional arguments for the Popen constructor

    Returns:
        Number of failed commands

    """
    for a in ['stdout', 'stderr']:
        if a in kwargs:
            raise ValueError('%s argument not allowed, '
                             'it will be overridden.' % a)
    error = 0

    if callback is None:
        if raise_on_error:
            # Raise on command failure
            def callback(r, c, o):
                raise CalledProcessError(r, c, o)
        else:
            # Continue commands execution
            callback = lambda r,c,o: True
    elif not callable(callback):
        raise ValueError('callback argument must be callable')

    # Manage stderr
    if separate_stderr:
        _stderr = subprocess.PIPE
        _get_output = lambda o,e: (o,e)
    else:
        _stderr = subprocess.STDOUT
        _get_output = lambda o,e: o

    # Iterate over commands
    for cmd in cmds:
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                   stderr=_stderr, shell=shell, **kwargs)
        output = _get_output(*process.communicate())
        retcode = process.poll()
        if retcode:
            error += 1
            if not callback(retcode, cmd, output):
                break
    return error
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    #url(r'^$', 'taxiexpress.views.index', name='index'),
    url(r'^client/login', 'taxiexpress.views.loginUser', name='loginUser'),
    url(r'^client/register', 'taxiexpress.views.registerUser', name='registerUser'),
    url(r'^client/validate', 'taxiexpress.views.validateUser', name='validateUser'),
    url(r'^client/recovervalidationcode', 'taxiexpress.views.recoverValidationCodeCustomer', name='recoverValidationCodeCustomer'),
    url(r'^client/gettaxi', 'taxiexpress.views.getClosestTaxi', name='getClosestTaxi'),
    url(r'^client/getselectedtaxi', 'taxiexpress.views.getSelectedTaxi', name='getSelectedTaxi'),
    url(r'^client/travelpaid', 'taxiexpress.views.travelPaid', name='travelPaid'),
    url(r'^client/canceltravel', 'taxiexpress.views.cancelTravel', name='cancelTravel'),
    url(r'^client/getneartaxies', 'taxiexpress.views.getNearestTaxies', name='getNearestTaxies'),
    url(r'^client/changedetails', 'taxiexpress.views.updateProfile', name='updateProfile'),
    url(r'^client/changepassword', 'taxiexpress.views.changePassword', name='changePassword'),
    url(r'^client/changefilters', 'taxiexpress.views.updateFilters', name='updateFilters'),
    url(r'^client/addfavorite', 'taxiexpress.views.addFavoriteDriver', name='addFavoriteDriver'),
    url(r'^client/removefavorite', 'taxiexpress.views.removeFavoriteDriver', name='removeFavoriteDriver'),
    url(r'^client/getlasttravel', 'taxiexpress.views.getLastTravel', name='getLastTravel'),
    url(r'^client/voteDriver', 'taxiexpress.views.voteDriver', name='voteDriver'),
    url(r'^driver/login', 'taxiexpress.views.loginDriver', name='loginDriver'),
    url(r'^driver/updatedriverposition', 'taxiexpress.views.updateDriverPosition', name='updateDriverPosition'),
    url(r'^driver/updatedriveravailable', 'taxiexpress.views.updateDriverAvailable', name='updateDriverAvailable'),
    url(r'^driver/recovervalidationcode', 'taxiexpress.views.recoverValidationCodeDriver', name='recoverValidationCodeDriver'),
    url(r'^driver/accepttravel', 'taxiexpress.views.acceptTravel', name='acceptTravel'),
    url(r'^driver/travelstarted', 'taxiexpress.views.travelStarted', name='travelStarted'),
    url(r'^driver/travelcompleted', 'taxiexpress.views.travelCompleted', name='travelCompleted'),
    url(r'^driver/voteClient', 'taxiexpress.views.voteClient', name='voteClient'),
    url(r'^test', 'taxiexpress.views.testPush', name='test'),
    url(r'^loaddata', 'taxiexpress.views.loadData', name='loaddata'),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    url(r'^recoverpassword', 'taxiexpress.views.recoverPassword', name='recoverPassword'),

    # url(r'^server/', include('server.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),

        #url para la Web
    url(r'^$', 'web.views.index', name='index'),
    url(r'^map', 'web.views.map', name='map'),
    url(r'^client', 'web.views.client', name='client'),
    url(r'^driver', 'web.views.driver', name='driver'),
    url(r'^faq', 'web.views.faq', name='faq'),
    url(r'^legalnotice', 'web.views.legalnotice', name='legalnotice'),
    url(r'^register', 'web.views.register', name='register'),
    url(r'^mantclient_data', 'web.views.mantclient_data', name='mantclient_data'),
    url(r'^mantclient_changepassword', 'web.views.mantclient_changePassword', name='mantclient_changePassword'),
    url(r'^mantclient_preferences', 'web.views.mantclient_preferences', name='mantclient_preferences'),
    url(r'^mantdriver_data', 'web.views.mantdriver_data', name='mantdriver_data'),
    url(r'^mantdriver_changepassword', 'web.views.mantdriver_changePassword', name='mantdriver_changePassword'),
    url(r'^mantdriver_car', 'web.views.mantdriver_car', name='mantdriver_car'),
    url(r'^mantdriver_bankaccount', 'web.views.mantdriver_bankAccount', name='mantdriver_bankAccount'),
    url(r'^mantdriver_travelgraphic', 'web.views.mantdriver_TravelGraphic', name='mantdriver_TravelGraphic'),
    url(r'^termsofuse', 'web.views.termsofuse', name='termsofuse'),
    url(r'^validatecode', 'web.views.validateCode', name='validateCode'),
    url(r'^logout', 'web.views.logout', name='logout'),
    url(r'^updateprofileuserweb', 'web.views.updateProfileUserWeb', name='updateProfileUserWeb'),
    url(r'^updateprofiledriverweb', 'web.views.updateProfileDriverWeb', name='updateProfileDriverWeb'),
    url(r'^updatecarweb', 'web.views.updateCarWeb', name='updateCarWeb'),
    url(r'^recovervalidationcode', 'web.views.recoverValidationCodeWeb', name='recoverValidationCodeWeb'),
    url(r'^contact', 'web.views.contact', name='contact'),
    url(r'^statistics/gettravelsbymonth', 'web.views.getTravelsByMonth', name='getTravelsByMonth'),
    url(r'^getcountries', 'web.views.getCountries', name='getCountries'),
    url(r'^getstates', 'web.views.getStates', name='getStates'),
    url(r'^getcities', 'web.views.getCities', name='getCities'),
    url(r'^rememberpassword', 'web.views.rememberPassword', name='rememberPassword'),
	url(r'^i18n/', include('django.conf.urls.i18n')),
    url(r'^statistics/gettravelsbymonth', 'web.views.getTravelsByMonth', name='getTravelsByMonth'),
    url(r'^statistics/gettravelsbyyear', 'web.views.getTravelsByYear', name='getTravelsByYear'),
    url(r'^statistics/gettravelsbyhour', 'web.views.getTravelsByHour', name='getTravelsByHour'),
    url(r'^statistics/gettravelsbyday', 'web.views.getTravelsByDay', name='getTravelsByDay'),
   )
<commit_msg>Url correction<commit_after>from django.conf.urls import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    #url(r'^$', 'taxiexpress.views.index', name='index'),
    url(r'^client/login', 'taxiexpress.views.loginUser', name='loginUser'),
    url(r'^client/register', 'taxiexpress.views.registerUser', name='registerUser'),
    url(r'^client/validate', 'taxiexpress.views.validateUser', name='validateUser'),
    url(r'^client/recovervalidationcode', 'taxiexpress.views.recoverValidationCodeCustomer', name='recoverValidationCodeCustomer'),
    url(r'^client/gettaxi', 'taxiexpress.views.getClosestTaxi', name='getClosestTaxi'),
    url(r'^client/getselectedtaxi', 'taxiexpress.views.getSelectedTaxi', name='getSelectedTaxi'),
    url(r'^client/travelpaid', 'taxiexpress.views.travelPaid', name='travelPaid'),
    url(r'^client/canceltravel', 'taxiexpress.views.cancelTravel', name='cancelTravel'),
    url(r'^client/getneartaxies', 'taxiexpress.views.getNearestTaxies', name='getNearestTaxies'),
    url(r'^client/changedetails', 'taxiexpress.views.updateProfile', name='updateProfile'),
    url(r'^client/changepassword', 'taxiexpress.views.changePassword', name='changePassword'),
    url(r'^client/changefilters', 'taxiexpress.views.updateFilters', name='updateFilters'),
    url(r'^client/addfavorite', 'taxiexpress.views.addFavoriteDriver', name='addFavoriteDriver'),
    url(r'^client/removefavorite', 'taxiexpress.views.removeFavoriteDriver', name='removeFavoriteDriver'),
    url(r'^client/getlasttravel', 'taxiexpress.views.getLastTravel', name='getLastTravel'),
    url(r'^client/votedriver', 'taxiexpress.views.voteDriver', name='voteDriver'),
    url(r'^driver/login', 'taxiexpress.views.loginDriver', name='loginDriver'),
    url(r'^driver/updatedriverposition', 'taxiexpress.views.updateDriverPosition', name='updateDriverPosition'),
    url(r'^driver/updatedriveravailable', 'taxiexpress.views.updateDriverAvailable', name='updateDriverAvailable'),
    url(r'^driver/recovervalidationcode', 'taxiexpress.views.recoverValidationCodeDriver', name='recoverValidationCodeDriver'),
    url(r'^driver/accepttravel', 'taxiexpress.views.acceptTravel', name='acceptTravel'),
    url(r'^driver/travelstarted', 'taxiexpress.views.travelStarted', name='travelStarted'),
    url(r'^driver/travelcompleted', 'taxiexpress.views.travelCompleted', name='travelCompleted'),
    url(r'^driver/votecustomer', 'taxiexpress.views.voteCustomer', name='voteCustomer'),
    url(r'^test', 'taxiexpress.views.testPush', name='test'),
    url(r'^loaddata', 'taxiexpress.views.loadData', name='loaddata'),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    url(r'^recoverpassword', 'taxiexpress.views.recoverPassword', name='recoverPassword'),

    # url(r'^server/', include('server.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),

        #url para la Web
    url(r'^$', 'web.views.index', name='index'),
    url(r'^map', 'web.views.map', name='map'),
    url(r'^client', 'web.views.client', name='client'),
    url(r'^driver', 'web.views.driver', name='driver'),
    url(r'^faq', 'web.views.faq', name='faq'),
    url(r'^legalnotice', 'web.views.legalnotice', name='legalnotice'),
    url(r'^register', 'web.views.register', name='register'),
    url(r'^mantclient_data', 'web.views.mantclient_data', name='mantclient_data'),
    url(r'^mantclient_changepassword', 'web.views.mantclient_changePassword', name='mantclient_changePassword'),
    url(r'^mantclient_preferences', 'web.views.mantclient_preferences', name='mantclient_preferences'),
    url(r'^mantdriver_data', 'web.views.mantdriver_data', name='mantdriver_data'),
    url(r'^mantdriver_changepassword', 'web.views.mantdriver_changePassword', name='mantdriver_changePassword'),
    url(r'^mantdriver_car', 'web.views.mantdriver_car', name='mantdriver_car'),
    url(r'^mantdriver_bankaccount', 'web.views.mantdriver_bankAccount', name='mantdriver_bankAccount'),
    url(r'^mantdriver_travelgraphic', 'web.views.mantdriver_TravelGraphic', name='mantdriver_TravelGraphic'),
    url(r'^termsofuse', 'web.views.termsofuse', name='termsofuse'),
    url(r'^validatecode', 'web.views.validateCode', name='validateCode'),
    url(r'^logout', 'web.views.logout', name='logout'),
    url(r'^updateprofileuserweb', 'web.views.updateProfileUserWeb', name='updateProfileUserWeb'),
    url(r'^updateprofiledriverweb', 'web.views.updateProfileDriverWeb', name='updateProfileDriverWeb'),
    url(r'^updatecarweb', 'web.views.updateCarWeb', name='updateCarWeb'),
    url(r'^recovervalidationcode', 'web.views.recoverValidationCodeWeb', name='recoverValidationCodeWeb'),
    url(r'^contact', 'web.views.contact', name='contact'),
    url(r'^statistics/gettravelsbymonth', 'web.views.getTravelsByMonth', name='getTravelsByMonth'),
    url(r'^getcountries', 'web.views.getCountries', name='getCountries'),
    url(r'^getstates', 'web.views.getStates', name='getStates'),
    url(r'^getcities', 'web.views.getCities', name='getCities'),
    url(r'^rememberpassword', 'web.views.rememberPassword', name='rememberPassword'),
	url(r'^i18n/', include('django.conf.urls.i18n')),
    url(r'^statistics/gettravelsbymonth', 'web.views.getTravelsByMonth', name='getTravelsByMonth'),
    url(r'^statistics/gettravelsbyyear', 'web.views.getTravelsByYear', name='getTravelsByYear'),
    url(r'^statistics/gettravelsbyhour', 'web.views.getTravelsByHour', name='getTravelsByHour'),
    url(r'^statistics/gettravelsbyday', 'web.views.getTravelsByDay', name='getTravelsByDay'),
   )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in module root
# directory
##############################################################################
from openerp import models, fields, api

class SaleOrderLine(models.Model):

    _inherit = 'sale.order.line'

    uom_unit_ids = fields.Many2many('product.uom', compute='_get_units')

    @api.one
    @api.depends('product_id')
    def _get_units(self):
        self.uom_unit_ids = self.get_product_uoms(self.product_id)

    @api.model
    def get_product_uoms(self, product):
        """
        if product has uoms configured, we use them
        if not, we choose all uoms from uom_id category (first the product uom)
        """
        # we can not use product.mapped('sale_uom_ids.uom_id') becuase it loose
        # order of sale_uom_ids
        return (
            self.env['product.uom'].browse(
                [x.uom_id.id for x in product.sale_uom_ids]) or
            (product.uom_id + self.env['product.uom'].search([
                ('category_id', '=', product.uom_id.category_id.id),
                ('id', '!=', product.uom_id.id),
                ]))
            )

    @api.multi
    def product_id_change(
            self, pricelist, product, qty=0,
            uom=False, qty_uos=0, uos=False, name='', partner_id=False,
            lang=False, update_tax=True, date_order=False, packaging=False,
            fiscal_position=False, flag=False):
        # because sale_stock module delete uom when colling this method, we
        # add it in context con module 'sale_stock_product_uom_prices'
        if not uom:
            uom = self._context.get('preserve_uom', False)
        res = super(SaleOrderLine, self).product_id_change(
            pricelist, product, qty=qty, uom=uom,
            qty_uos=qty_uos, uos=uos, name=name, partner_id=partner_id,
            lang=lang, update_tax=update_tax, date_order=date_order,
            packaging=packaging, fiscal_position=fiscal_position,
            flag=flag)

        if product:
            context_partner = {'lang': lang, 'partner_id': partner_id}
            product = self.env['product.product'].with_context(
                context_partner).browse(
                product)

            # we can use line on self but we should use self.ensure_one()
            sale_product_uoms = self.get_product_uoms(product)
            if sale_product_uoms:
                if not uom:
                    uom_id = sale_product_uoms[0].id
                    res['value']['product_uom'] = uom_id

                # we do this because odoo overwrite view domain
                if 'domain' not in res:
                    res['domain'] = {}
                res['domain']['product_uom'] = [
                    ('id', 'in', sale_product_uoms.ids)]
        return res
<commit_msg>FIX prod sale uom<commit_after># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in module root
# directory
##############################################################################
from openerp import models, fields, api

class SaleOrderLine(models.Model):

    _inherit = 'sale.order.line'

    uom_unit_ids = fields.Many2many('product.uom', compute='_get_units')

    @api.one
    @api.depends('product_id')
    def _get_units(self):
        self.uom_unit_ids = self.get_product_uoms(self.product_id)

    @api.model
    def get_product_uoms(self, product):
        """
        if product has uoms configured, we use them
        if not, we choose all uoms from uom_id category (first the product uom)
        """
        # we can not use product.mapped('sale_uom_ids.uom_id') becuase it loose
        # order of sale_uom_ids
        return (
            self.env['product.uom'].browse(
                [x.uom_id.id for x in product.sale_uom_ids]) or
            (product.uom_id + self.env['product.uom'].search([
                ('category_id', '=', product.uom_id.category_id.id),
                ('id', '!=', product.uom_id.id),
                ]))
            )

    @api.multi
    def product_id_change(
            self, pricelist, product, qty=0,
            uom=False, qty_uos=0, uos=False, name='', partner_id=False,
            lang=False, update_tax=True, date_order=False, packaging=False,
            fiscal_position=False, flag=False):
        # because sale_stock module delete uom when colling this method, we
        # add it in context con module 'sale_stock_product_uom_prices'
        if not uom:
            uom = self._context.get('preserve_uom', False)
        product_uom_domain = None
        if product:
            context_partner = {'lang': lang, 'partner_id': partner_id}
            product_obj = self.env['product.product'].with_context(
                context_partner).browse(
                product)

            # we can use line on self but we should use self.ensure_one()
            sale_product_uoms = self.get_product_uoms(product_obj)
            if sale_product_uoms:
                if not uom:
                    uom = sale_product_uoms[0].id

                # we do this because odoo overwrite view domain
                product_uom_domain = [('id', 'in', sale_product_uoms.ids)]
        res = super(SaleOrderLine, self).product_id_change(
            pricelist, product, qty=qty, uom=uom,
            qty_uos=qty_uos, uos=uos, name=name, partner_id=partner_id,
            lang=lang, update_tax=update_tax, date_order=date_order,
            packaging=packaging, fiscal_position=fiscal_position,
            flag=flag)
        if product_uom_domain:
            if 'domain' not in res:
                res['domain'] = {}
            res['domain']['product_uom'] = product_uom_domain
        return res
<|endoftext|>


<commit_before>#!/usr/bin/python2

import cairo, colorsys
import csv, math, json, time, sys, gzip
from contextlib import closing

class StationCoverage:
    def __init__(self, name, lat, lon, privacy=False, binsize=0.05, alt_mode=False, is_station=True):
        self.name = name
        self.alt_mode = alt_mode
        if lat:
            if privacy:
                self.station_lat = self.station_lon = None
            else:
                self.station_lat = round(lat / (binsize/2)) * (binsize/2)
                self.station_lon = round(lon / (binsize/2)) * (binsize/2)
        else:
            self.station_lat = self.station_lon = None

        self.binsize = binsize
        self.bins = {}
        self.min_lat = self.max_lat = self.min_lon = self.max_lon = None
        self.max_count = None
        self.is_station = is_station

    def add_position(self, lat, lon, alt, err_est):
        bin_lat = math.floor(lat / self.binsize) * self.binsize
        bin_lon = math.floor(lon / self.binsize) * self.binsize

        if self.min_lat is None or bin_lat < self.min_lat: self.min_lat = bin_lat
        if self.min_lon is None or bin_lon < self.min_lon: self.min_lon = bin_lon
        if self.max_lat is None or bin_lat > self.max_lat: self.max_lat = bin_lat
        if self.max_lon is None or bin_lon > self.max_lon: self.max_lon = bin_lon

        bin_key = (bin_lat, bin_lon)
        if self.alt_mode:
            data = self.bins.setdefault(bin_key, [1,99999.0])
            data[1] = min(data[1], alt)
        else:
            data = self.bins.setdefault(bin_key, [0,0.0])
            data[0] += 1
            data[1] += err_est
            if self.max_count is None or data[0] > self.max_count: self.max_count = data[0]

    def write(self, basedir, pngfile, metafile, pixels_per_degree=None):
        if len(self.bins) == 0: return

        if not pixels_per_degree: pixels_per_degree = math.ceil(4.0 / self.binsize)

        min_lon = self.min_lon
        min_lat = self.min_lat
        max_lat = self.max_lat + self.binsize
        max_lon = self.max_lon + self.binsize
        binsize = self.binsize

        lon_span = (max_lon-min_lon)
        lat_span = (max_lat-min_lat)
        xsize = int(math.ceil(lon_span*pixels_per_degree))
        ysize = int(math.ceil(lat_span*pixels_per_degree))

        surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, xsize, ysize)

        cc = cairo.Context(surface)

        cc.set_operator(cairo.OPERATOR_SOURCE)
        cc.set_antialias(cairo.ANTIALIAS_NONE)
        cc.scale(1.0 * xsize / lon_span, -1.0 * ysize / lat_span)
        cc.translate(-min_lon, -max_lat)

        # transparent background
        cc.set_source(cairo.SolidPattern(0,0,0,0))
        cc.paint()

        # draw bins
        for (bin_lat,bin_lon), (count,val) in self.bins.items():
            a = 0.75

            if self.alt_mode:
                if val < 2000:
                    hue = 20
                elif val < 10000:
                    hue = 20 + 120.0 * (val-2000) / 8000.0
                elif val < 40000:
                    hue = 140 + 160 * (val-10000) / 30000.0
                else:
                    hue = 300
            else:
                err_est = val / count
                hue = 150.0 - 150 * ((err_est / 5000.0)**0.5)
                if hue < 0:
                    hue = 0

            r,g,b = colorsys.hls_to_rgb(hue/360.0, 0.5, 1.0)
            cc.set_source(cairo.SolidPattern(r,g,b,a))

            cc.move_to(bin_lon, bin_lat)
            cc.line_to(bin_lon + binsize, bin_lat)
            cc.line_to(bin_lon + binsize, bin_lat + binsize)
            cc.line_to(bin_lon, bin_lat + binsize)
            cc.close_path()
            cc.fill()

        surface.write_to_png(basedir + '/' + pngfile)

        print >>metafile, """
coverage['{name}'] = {{
  name:    '{name}',
  lat:     {station_lat},
  lon:     {station_lon},
  min_lat: {min_lat},
  min_lon: {min_lon},
  max_lat: {max_lat},
  max_lon: {max_lon},
  image:   '{pngfile}',
  is_station: {is_station}
}};""".format(name = self.name,
              station_lat = (self.station_lat is None) and 'null' or self.station_lat,
              station_lon = (self.station_lon is None) and 'null' or self.station_lon,
              min_lat = self.min_lat,
              min_lon = self.min_lon,
              max_lat = max_lat,
              max_lon = max_lon,
              pngfile = pngfile,
              is_station = self.is_station and "true" or "false")

def multiopen(path):
    if path[-3:] == '.gz':
        return gzip.open(path, 'rb')
    else:
        return open(path, 'rb')

def plot_from_datafile(csvfile, jsonfile, outdir):
    station_coverage = {
        '*' : StationCoverage('all', None, None, is_station=False),
        '4+' : StationCoverage('4plus', None, None, is_station=False),
        '5+' : StationCoverage('5plus', None, None, is_station=False),
        '6+' : StationCoverage('6plus', None, None, is_station=False),
        '10000-' : StationCoverage('below10000', None, None, is_station=False),
        '18000-' : StationCoverage('below18000', None, None, is_station=False),
        'byalt' : StationCoverage('byalt', None, None, alt_mode=True, is_station=False)
    }

    with closing(multiopen(jsonfile)) as f:
        station_data = json.load(f)

    for station_name, station_pos in station_data.items():
        station_coverage[station_name] = StationCoverage(station_name, station_pos['lat'], station_pos['lon'], privacy=station_pos['privacy'])

    first = last = None
    num_positions = 0
    with closing(multiopen(csvfile)) as f:
        reader = csv.reader(f)
        for row in reader:
            try:
                t,addr,callsign,squawk,lat,lon,alt,err_est,nstations,ndistinct,stationlist = row[:11]
            except ValueError as e:
                print 'row', reader.line_num, 'failed: ', str(e)
                print repr(row)
                continue

            t = float(t)
            lat = float(lat)
            lon = float(lon)
            alt = float(alt)
            err_est = max(0, float(err_est))
            nstations = int(nstations)
            ndistinct = int(ndistinct)

            if not first:
                first = last = t
            else:
                first = min(first,t)
                last = max(last,t)

            station_coverage['*'].add_position(lat,lon,alt,err_est)
            station_coverage['byalt'].add_position(lat,lon,alt,err_est)
            if nstations >= 4:
                station_coverage['4+'].add_position(lat,lon,alt,err_est)
            if nstations >= 5:
                station_coverage['5+'].add_position(lat,lon,alt,err_est)
            if nstations >= 6:
                station_coverage['6+'].add_position(lat,lon,alt,err_est)
            if alt <= 10000:
                station_coverage['10000-'].add_position(lat,lon,alt,err_est)
            if alt <= 18000:
                station_coverage['18000-'].add_position(lat,lon,alt,err_est)

            for s in stationlist.split(','):
                sc = station_coverage.get(s)
                if not sc:
                    sc = station_coverage[s] = StationCoverage(s, None, None)
                sc.add_position(lat, lon, alt, err_est)

            num_positions += 1

    with closing(open(outdir + '/data.js', 'w')) as metafile:
        print >>metafile, "var first_position = '{d}';".format(d = time.strftime("%Y/%m/%d %H:%M:%S UTC", time.gmtime(first)))
        print >>metafile, "var last_position = '{d}';".format(d = time.strftime("%Y/%m/%d %H:%M:%S UTC", time.gmtime(last)))
        print >>metafile, "var num_positions = {n};".format(n = num_positions)
        print >>metafile, "var coverage = {};"
        for sc in station_coverage.values():
            pngfile = 'coverage_{n}.png'.format(n=sc.name)
            #print 'Writing', pngfile
            sc.write(basedir = outdir, pngfile = pngfile, metafile = metafile)

if __name__ == '__main__':
    plot_from_datafile(csvfile=sys.argv[1], jsonfile=sys.argv[2], outdir=sys.argv[3])
<commit_msg>Cairo lib changed to easirer installable cairocffi<commit_after>#!/usr/bin/python2

import cairocffi as cairo, colorsys
import csv, math, json, time, sys, gzip
from contextlib import closing

class StationCoverage:
    def __init__(self, name, lat, lon, privacy=False, binsize=0.05, alt_mode=False, is_station=True):
        self.name = name
        self.alt_mode = alt_mode
        if lat:
            if privacy:
                self.station_lat = self.station_lon = None
            else:
                self.station_lat = round(lat / (binsize/2)) * (binsize/2)
                self.station_lon = round(lon / (binsize/2)) * (binsize/2)
        else:
            self.station_lat = self.station_lon = None

        self.binsize = binsize
        self.bins = {}
        self.min_lat = self.max_lat = self.min_lon = self.max_lon = None
        self.max_count = None
        self.is_station = is_station

    def add_position(self, lat, lon, alt, err_est):
        bin_lat = math.floor(lat / self.binsize) * self.binsize
        bin_lon = math.floor(lon / self.binsize) * self.binsize

        if self.min_lat is None or bin_lat < self.min_lat: self.min_lat = bin_lat
        if self.min_lon is None or bin_lon < self.min_lon: self.min_lon = bin_lon
        if self.max_lat is None or bin_lat > self.max_lat: self.max_lat = bin_lat
        if self.max_lon is None or bin_lon > self.max_lon: self.max_lon = bin_lon

        bin_key = (bin_lat, bin_lon)
        if self.alt_mode:
            data = self.bins.setdefault(bin_key, [1,99999.0])
            data[1] = min(data[1], alt)
        else:
            data = self.bins.setdefault(bin_key, [0,0.0])
            data[0] += 1
            data[1] += err_est
            if self.max_count is None or data[0] > self.max_count: self.max_count = data[0]

    def write(self, basedir, pngfile, metafile, pixels_per_degree=None):
        if len(self.bins) == 0: return

        if not pixels_per_degree: pixels_per_degree = math.ceil(4.0 / self.binsize)

        min_lon = self.min_lon
        min_lat = self.min_lat
        max_lat = self.max_lat + self.binsize
        max_lon = self.max_lon + self.binsize
        binsize = self.binsize

        lon_span = (max_lon-min_lon)
        lat_span = (max_lat-min_lat)
        xsize = int(math.ceil(lon_span*pixels_per_degree))
        ysize = int(math.ceil(lat_span*pixels_per_degree))

        surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, xsize, ysize)

        cc = cairo.Context(surface)

        cc.set_operator(cairo.OPERATOR_SOURCE)
        cc.set_antialias(cairo.ANTIALIAS_NONE)
        cc.scale(1.0 * xsize / lon_span, -1.0 * ysize / lat_span)
        cc.translate(-min_lon, -max_lat)

        # transparent background
        cc.set_source(cairo.SolidPattern(0,0,0,0))
        cc.paint()

        # draw bins
        for (bin_lat,bin_lon), (count,val) in self.bins.items():
            a = 0.75

            if self.alt_mode:
                if val < 2000:
                    hue = 20
                elif val < 10000:
                    hue = 20 + 120.0 * (val-2000) / 8000.0
                elif val < 40000:
                    hue = 140 + 160 * (val-10000) / 30000.0
                else:
                    hue = 300
            else:
                err_est = val / count
                hue = 150.0 - 150 * ((err_est / 5000.0)**0.5)
                if hue < 0:
                    hue = 0

            r,g,b = colorsys.hls_to_rgb(hue/360.0, 0.5, 1.0)
            cc.set_source(cairo.SolidPattern(r,g,b,a))

            cc.move_to(bin_lon, bin_lat)
            cc.line_to(bin_lon + binsize, bin_lat)
            cc.line_to(bin_lon + binsize, bin_lat + binsize)
            cc.line_to(bin_lon, bin_lat + binsize)
            cc.close_path()
            cc.fill()

        surface.write_to_png(basedir + '/' + pngfile)

        print >>metafile, """
coverage['{name}'] = {{
  name:    '{name}',
  lat:     {station_lat},
  lon:     {station_lon},
  min_lat: {min_lat},
  min_lon: {min_lon},
  max_lat: {max_lat},
  max_lon: {max_lon},
  image:   '{pngfile}',
  is_station: {is_station}
}};""".format(name = self.name,
              station_lat = (self.station_lat is None) and 'null' or self.station_lat,
              station_lon = (self.station_lon is None) and 'null' or self.station_lon,
              min_lat = self.min_lat,
              min_lon = self.min_lon,
              max_lat = max_lat,
              max_lon = max_lon,
              pngfile = pngfile,
              is_station = self.is_station and "true" or "false")

def multiopen(path):
    if path[-3:] == '.gz':
        return gzip.open(path, 'rb')
    else:
        return open(path, 'rb')

def plot_from_datafile(csvfile, jsonfile, outdir):
    station_coverage = {
        '*' : StationCoverage('all', None, None, is_station=False),
        '4+' : StationCoverage('4plus', None, None, is_station=False),
        '5+' : StationCoverage('5plus', None, None, is_station=False),
        '6+' : StationCoverage('6plus', None, None, is_station=False),
        '10000-' : StationCoverage('below10000', None, None, is_station=False),
        '18000-' : StationCoverage('below18000', None, None, is_station=False),
        'byalt' : StationCoverage('byalt', None, None, alt_mode=True, is_station=False)
    }

    with closing(multiopen(jsonfile)) as f:
        station_data = json.load(f)

    for station_name, station_pos in station_data.items():
        station_coverage[station_name] = StationCoverage(station_name, station_pos['lat'], station_pos['lon'], privacy=station_pos['privacy'])

    first = last = None
    num_positions = 0
    with closing(multiopen(csvfile)) as f:
        reader = csv.reader(f)
        for row in reader:
            try:
                t,addr,callsign,squawk,lat,lon,alt,err_est,nstations,ndistinct,stationlist = row[:11]
            except ValueError as e:
                print 'row', reader.line_num, 'failed: ', str(e)
                print repr(row)
                continue

            t = float(t)
            lat = float(lat)
            lon = float(lon)
            alt = float(alt)
            err_est = max(0, float(err_est))
            nstations = int(nstations)
            ndistinct = int(ndistinct)

            if not first:
                first = last = t
            else:
                first = min(first,t)
                last = max(last,t)

            station_coverage['*'].add_position(lat,lon,alt,err_est)
            station_coverage['byalt'].add_position(lat,lon,alt,err_est)
            if nstations >= 4:
                station_coverage['4+'].add_position(lat,lon,alt,err_est)
            if nstations >= 5:
                station_coverage['5+'].add_position(lat,lon,alt,err_est)
            if nstations >= 6:
                station_coverage['6+'].add_position(lat,lon,alt,err_est)
            if alt <= 10000:
                station_coverage['10000-'].add_position(lat,lon,alt,err_est)
            if alt <= 18000:
                station_coverage['18000-'].add_position(lat,lon,alt,err_est)

            for s in stationlist.split(','):
                sc = station_coverage.get(s)
                if not sc:
                    sc = station_coverage[s] = StationCoverage(s, None, None)
                sc.add_position(lat, lon, alt, err_est)

            num_positions += 1

    with closing(open(outdir + '/data.js', 'w')) as metafile:
        print >>metafile, "var first_position = '{d}';".format(d = time.strftime("%Y/%m/%d %H:%M:%S UTC", time.gmtime(first)))
        print >>metafile, "var last_position = '{d}';".format(d = time.strftime("%Y/%m/%d %H:%M:%S UTC", time.gmtime(last)))
        print >>metafile, "var num_positions = {n};".format(n = num_positions)
        print >>metafile, "var coverage = {};"
        for sc in station_coverage.values():
            pngfile = 'coverage_{n}.png'.format(n=sc.name)
            #print 'Writing', pngfile
            sc.write(basedir = outdir, pngfile = pngfile, metafile = metafile)

if __name__ == '__main__':
    plot_from_datafile(csvfile=sys.argv[1], jsonfile=sys.argv[2], outdir=sys.argv[3])
<|endoftext|>


<commit_before>
# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Created by Disa Mhembere
# Email: disa@jhu.edu

from computation.utils.loadAdjMatrix import loadAdjMat
from glob import glob
import os
import scipy.io as sio
import pdb
import sys

def load_and_store(dirg):
  """"
  load up and LCC adjacency matrix and save it elsewhere

  *NOTE: This is not a general script and is specific to the bg1:/data/public/MR/MIGRAINE data
  Positional Args:
  ===============
  dirg - the directory with a graph
  """
  print "** Processing dataset: %s ... **\n" % dirg

  graphs = glob( os.path.join(dirg, "*"))

  if dirg.endswith("//"): dirg = dirg[:-1]
  base_dir = os.path.dirname(dirg)
  save_dir = os.path.join(base_dir, "big_lcc_graphs")

  if not os.path.exists(save_dir):
    print "Making %s ..." % save_dir
    os.makedirs(save_dir)

  for g_fn in graphs:
    print "\nProcessing %s ..." % g_fn

    fn_root = g_fn.split("/")[-1][:-13]
    lcc_fn = os.path.join(base_dir, "big_lcc", fn_root+"big_lcc.npy")

    if os.path.exists(g_fn) and os.path.exists(lcc_fn):
      g = loadAdjMat(g_fn, lcc_fn)
      fn = os.path.join(save_dir, fn_root+"big_lcc_adjmat")
      print "Saving %s ..." % fn
      sio.savemat( os.path.join(save_dir, fn), {"data":g} )

    else:
      if not os.path.exists(g_fn): print "Graph path %s does not exist ..." % g_fn
      if not os.path.exists(lcc_fn): print "Lcc path %s does not exist ..." % lcc_fn

  print "** Done with %s ** \n\n" % dirg

if __name__ == '__main__':
  if len(sys.argv) > 1:
    load_and_store(sys.argv[1])
  else:
    print "Please provide the name of the directory with LCC's to convert as the first arg!"<commit_msg>[Graph]: Interface fixes for testing<commit_after>
# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Created by Disa Mhembere
# Email: disa@jhu.edu

from computation.utils.loadAdjMatrix import loadAdjMat
from glob import glob
import os
import scipy.io as sio
import pdb
import sys

def load_and_store(dirg):
  """"
  load up and LCC adjacency matrix and save it elsewhere

  *NOTE: This is not a general script and is specific to the bg1:/data/public/MR/MIGRAINE data
  Positional Args:
  ===============
  dirg - the directory with a graph
  """
  print "** Processing dataset: %s ... **\n" % dirg

  graphs = glob( os.path.join(dirg, "*"))

  if dirg.endswith("//"): dirg = dirg[:-1]
  base_dir = os.path.dirname(dirg)
  save_dir = os.path.join(base_dir, "big_lcc_graphs")

  if not os.path.exists(save_dir):
    print "Making %s ..." % save_dir
    os.makedirs(save_dir)

  for g_fn in graphs:
    print "\nProcessing %s ..." % g_fn

    fn_root = g_fn.split("/")[-1][:-13]
    lcc_fn = os.path.join(base_dir, "big_lcc", fn_root+"big_lcc.npy")

    if os.path.exists(g_fn) and os.path.exists(lcc_fn):
      g = loadAdjMat(g_fn, lcc_fn)
      fn = os.path.join(save_dir, fn_root+"big_lcc_adjmat")
      print "Saving %s ..." % fn
      sio.savemat( os.path.join(save_dir, fn), {"data":g} )

    else:
      if not os.path.exists(g_fn): print "Graph path %s does not exist ..." % g_fn
      if not os.path.exists(lcc_fn): print "Lcc path %s does not exist ..." % lcc_fn

  print "** Done with %s ** \n\n" % dirg

if __name__ == '__main__':
  if len(sys.argv) > 1 sys.argv[1] != "-h":
    load_and_store(sys.argv[1])
  else:
    print "Please provide the name of the directory with LCC's to convert as the first arg!"
<|endoftext|>