<commit_before>from flask import Flask, Response

from nose.tools import raises

from .resolver import Pushrod, pushrod_view
from .formatters.base import formatter, UnformattedResponse, FormatterNotFound
from .formatters.json import json_formatter

from unittest import TestCase
import json

test_response = {
    "spam": "eggs",
    "yes": [
        "maybe",
        "no",
    ],
    "nonsense": (
        True,
        False,
        "FILE_NOT_FOUND",
        -5.4,
    ),
}

@formatter('repr')
def repr_formatter(unformatted):
    return unformatted.formatted(repr(unformatted.response), "text/plain")

class PushrodTestCase(TestCase):
    def setUp(self):
        app = Flask(__name__)
        pushrod = Pushrod(app)

        pushrod.register_formatter(repr_formatter)
        pushrod.default_formatter = repr_formatter

        self.app = app
        self.client = app.test_client()

class PushrodResolverTestCase(PushrodTestCase):
    def test_raw_dict_response_data(self):
        @self.app.route("/raw_dict")
        @pushrod_view()
        def test_view_raw_dict():
            return test_response

        test_view_raw_dict_response = self.client.get("/raw_dict")
        assert test_view_raw_dict_response.status_code == 200
        assert test_view_raw_dict_response.data == repr(test_response)

    def test_response_data(self):
        @self.app.route("/200_response")
        @pushrod_view()
        def test_view_200_view():
            return UnformattedResponse(test_response)

        @self.app.route("/404_response")
        @pushrod_view()
        def test_view_404_view():
            return UnformattedResponse(test_response, 404)

        test_view_200_response = self.client.get("/200_response")
        assert test_view_200_response.status_code == 200
        assert test_view_200_response.data == repr(test_response)

        test_view_404_response = self.client.get("/404_response")
        assert test_view_404_response.status_code == 404
        assert test_view_404_response.data == repr(test_response)

    def test_response_bypass(self):
        @self.app.route("/raw_string")
        @pushrod_view()
        def test_raw_string_view():
            return "test"

        @self.app.route("/regular_response")
        @pushrod_view()
        def test_regular_response_view():
            return Response("test")

        test_raw_string_response = self.client.get("/raw_string")
        assert test_raw_string_response.status_code == 200
        assert test_raw_string_response.data == "test"

        test_regular_response_response = self.client.get("/regular_response")
        assert test_regular_response_response.status_code == 200
        assert test_regular_response_response.data == "test"

    @raises(FormatterNotFound)
    def test_formatter_not_found(self):
        @self.app.route("/")
        @pushrod_view()
        def test_formatter_not_found_view():
            return {}

        response = self.client.get("/?format=none")
        assert response.status_code == 406, \
            "Status code returned from an unexistant formatter should be 406 (not acceptable)"

        with self.app.test_request_context("/?format=none"):
            self.app.pushrod.format_response(test_formatter_not_found_view(), "none")

class PushrodFormatterTestCase(PushrodTestCase):
    def test_json_formatter(self):
        regular = json.dumps(test_response)
        formatted = self.app.pushrod.format_response(
            test_response, json_formatter)

        assert regular == formatted.data

        json.loads(formatted.data)
<commit_msg>Intentionally failing test to make sure that Travis reports it properly<commit_after>from flask import Flask, Response

from nose.tools import raises

from .resolver import Pushrod, pushrod_view
from .formatters.base import formatter, UnformattedResponse, FormatterNotFound
from .formatters.json import json_formatter

from unittest import TestCase
import json

test_response = {
    "spam": "eggs",
    "yes": [
        "maybe",
        "no",
    ],
    "nonsense": (
        True,
        False,
        "FILE_NOT_FOUND",
        -5.4,
    ),
}

@formatter('repr')
def repr_formatter(unformatted):
    return unformatted.formatted(repr(unformatted.response), "text/plain")

class PushrodTestCase(TestCase):
    def setUp(self):
        app = Flask(__name__)
        pushrod = Pushrod(app)

        pushrod.register_formatter(repr_formatter)
        pushrod.default_formatter = repr_formatter

        self.app = app
        self.client = app.test_client()

class PushrodResolverTestCase(PushrodTestCase):
    def test_raw_dict_response_data(self):
        @self.app.route("/raw_dict")
        @pushrod_view()
        def test_view_raw_dict():
            return test_response

        test_view_raw_dict_response = self.client.get("/raw_dict")
        assert test_view_raw_dict_response.status_code == 200
        assert test_view_raw_dict_response.data == repr(test_response)

    def test_response_data(self):
        @self.app.route("/200_response")
        @pushrod_view()
        def test_view_200_view():
            return UnformattedResponse(test_response)

        @self.app.route("/404_response")
        @pushrod_view()
        def test_view_404_view():
            return UnformattedResponse(test_response, 404)

        test_view_200_response = self.client.get("/200_response")
        assert test_view_200_response.status_code == 200
        assert test_view_200_response.data == repr(test_response)

        test_view_404_response = self.client.get("/404_response")
        assert test_view_404_response.status_code == 404
        assert test_view_404_response.data == repr(test_response)

    def test_response_bypass(self):
        @self.app.route("/raw_string")
        @pushrod_view()
        def test_raw_string_view():
            return "test"

        @self.app.route("/regular_response")
        @pushrod_view()
        def test_regular_response_view():
            return Response("test")

        test_raw_string_response = self.client.get("/raw_string")
        assert test_raw_string_response.status_code == 200
        assert test_raw_string_response.data == "test"

        test_regular_response_response = self.client.get("/regular_response")
        assert test_regular_response_response.status_code == 200
        assert test_regular_response_response.data == "test"

    @raises(FormatterNotFound)
    def test_formatter_not_found(self):
        @self.app.route("/")
        @pushrod_view()
        def test_formatter_not_found_view():
            return {}

        response = self.client.get("/?format=none")
        assert response.status_code == 406, \
            "Status code returned from an unexistant formatter should be 406 (not acceptable)"

        with self.app.test_request_context("/?format=none"):
            pass
            #self.app.pushrod.format_response(test_formatter_not_found_view(), "none")

class PushrodFormatterTestCase(PushrodTestCase):
    def test_json_formatter(self):
        regular = json.dumps(test_response)
        formatted = self.app.pushrod.format_response(
            test_response, json_formatter)

        assert regular == formatted.data

        json.loads(formatted.data)
<|endoftext|>


<commit_before>""" __init__.py: Sets up Flaskpost and imports all additional pieces """
from flask_sqlalchemy import SQLAlchemy
from flask import Flask
from flask_login import LoginManager

app = Flask(__name__)
app.secret_key = "You should change this before using flaskpost."
#TODO: Find better home for database file?
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:////tmp/flaskpost.db"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False

db = SQLAlchemy(app)
from flaskpost.model import Metadata
db.create_all()

login_manager = LoginManager()
login_manager.init_app(app)

class ConfigSingleton:
    class __ConfigSingleton:
        needs_setup = None
        title = None
        def __init__(self, title=None, needs_setup=None):
            if title == None and needs_setup == None:
                self.needs_setup =\
                (Metadata.query.filter_by(key="setup_reverse_canary").first() ==\
                None)
                if not self.needs_setup:
                    self.title =\
                    Metadata.query.filter_by(key="blog_title").first().value
            else:
                self.title = title
                self.needs_setup = needs_setup

    instance = None
    def __init__(self):
        if not ConfigSingleton.instance:
            ConfigSingleton.instance = ConfigSingleton.__ConfigSingleton()

    def __getattr__(self, name):
        return getattr(self.instance, name)

    def update(self, title, needs_setup):
        self.instance = ConfigSingleton.__ConfigSingleton

import flaskpost.api
import flaskpost.paths
import flaskpost.decorators
<commit_msg>Added some documentation, fixed a bug<commit_after>""" __init__.py: Sets up Flaskpost and imports all additional pieces """
from flask_sqlalchemy import SQLAlchemy
from flask import Flask
from flask_login import LoginManager

app = Flask(__name__)
app.secret_key = "You should change this before using flaskpost."
#TODO: Find better home for database file?
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:////tmp/flaskpost.db"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False

db = SQLAlchemy(app)
from flaskpost.model import Metadata
db.create_all()

login_manager = LoginManager()
login_manager.init_app(app)

""" Singleton to load and contain the configuration of the blog. """
class ConfigSingleton:
    class __ConfigSingleton:
        needs_setup = None
        title = None
        def __init__(self, title=None, needs_setup=None):
            if title == None and needs_setup == None:
                self.needs_setup =\
                (Metadata.query.filter_by(key="setup_reverse_canary").first() ==\
                None)
                if not self.needs_setup:
                    self.title =\
                    Metadata.query.filter_by(key="blog_title").first().value
            else:
                self.title = title
                self.needs_setup = needs_setup

    instance = None
    def __init__(self):
        if not ConfigSingleton.instance:
            ConfigSingleton.instance = ConfigSingleton.__ConfigSingleton()

    # Pass all attribute access to the inner class.
    def __getattr__(self, name):
        return getattr(self.instance, name)

    def update(self, title, needs_setup):
        ConfigSingleton.instance = ConfigSingleton.__ConfigSingleton(title, needs_setup)

import flaskpost.api
import flaskpost.paths
import flaskpost.decorators
<|endoftext|>


<commit_before>import io
import json
from os import walk
from os.path import join
from tempfile import TemporaryDirectory
from urllib.request import urlopen
from zipfile import ZipFile

from 臺灣言語工具.解析整理.拆文分析器 import 拆文分析器
from 臺灣言語服務.models import 訓練過渡格式
from 匯入.指令 import 匯入枋模
from 臺灣言語工具.解析整理.解析錯誤 import 解析錯誤

class Command(匯入枋模):
    help = ''
    zip網址 = ''

    公家內容 = {
        '來源': '台語文數位典藏資料庫',
        '年代': '2006',
        '種類': '語句',
    }

    def 全部資料(self, *args, **參數):
        全部資料 = []
        匯入數量 = 0
        for 台文 in self._全部資料():
            全部資料.append(
                訓練過渡格式(
                    文本=台文.看分詞(),
                    **self.公家內容
                )
            )

            匯入數量 += 1
            if 匯入數量 % 1000 == 0:
                self.stdout.write('匯入 {} 筆'.format(匯入數量))

        return 全部資料

    def _全部資料(self):
        with TemporaryDirectory() as 資料夾:
            with urlopen(self.zip網址) as 網路檔:
                with io.BytesIO(網路檔.read()) as 檔:
                    with ZipFile(檔) as 資料:
                        資料.extractall(資料夾)
            yield from self.規目錄(
                join(資料夾, 'Ungian_2009_KIPsupin-master/', 'JSON格式資料')
            )

    def 規目錄(self, 語料資料夾):
        for 所在, _資料夾, 檔名陣列 in sorted(walk(語料資料夾)):
            for 檔名 in 檔名陣列:
                with open(join(所在, 檔名)) as 檔:
                    一檔 = json.load(檔)
                for 一篇 in 一檔['資料']:
                    一段 = 一篇['段']
                    for 漢羅, 羅 in 一段:
                        yield from self.轉物件(漢羅, 羅)

    def 轉物件(self, 漢羅, 羅):
        try:
            yield 拆文分析器.建立句物件(漢羅, 羅)
            return
        except 解析錯誤 as 錯誤:
            print(錯誤)
        try:
            yield 拆文分析器.建立句物件(羅)
            return
        except 解析錯誤 as 錯誤:
            print(錯誤)
<commit_msg>Ka thing-sik.<commit_after>import io
import json
from urllib.request import urlopen

from 臺灣言語工具.解析整理.拆文分析器 import 拆文分析器
from 臺灣言語服務.models import 訓練過渡格式
from 匯入.指令 import 匯入枋模
from 臺灣言語工具.解析整理.解析錯誤 import 解析錯誤

class Command(匯入枋模):
    help = 'http://xdcm.nmtl.gov.tw/dadwt/pbk.asp'
    json網址 = (
        'https://github.com/Taiwanese-Corpus/nmtl_2006_dadwt/'
        'raw/master/nmtl.json'
    )

    公家內容 = {
        '來源': '台語文數位典藏資料庫',
        '年代': '2006',
        '種類': '語句',
    }

    def 全部資料(self, *args, **參數):
        全部資料 = []
        匯入數量 = 0
        for 台文 in self._全部資料():
            全部資料.append(
                訓練過渡格式(
                    文本=台文.看分詞(),
                    **self.公家內容
                )
            )

            匯入數量 += 1
            if 匯入數量 % 1000 == 0:
                self.stdout.write('匯入 {} 筆'.format(匯入數量))

        return 全部資料

    def _全部資料(self):
        for 漢羅, 羅 in self._全部漢羅():
            try:
                yield from self.轉物件(漢羅, 羅)
            except ValueError:
                'https://github.com/i3thuan5/tai5-uan5_gian5-gi2_kang1-ku7/issues/566'
                pass

    def _全部漢羅(self):
        with urlopen(self.json網址) as 檔:
            with io.StringIO(檔.read().decode()) as 資料json:
                for phinn in json.load(資料json):
                    yield phinn['漢羅名'], phinn['全羅名']
                    yield phinn['漢羅標'], phinn['全羅標']
                    yield from phinn['資料']

    def 轉物件(self, 漢羅, 羅):
        try:
            yield 拆文分析器.建立句物件(漢羅, 羅)
            return
        except 解析錯誤 as 錯誤:
            print(錯誤)
        try:
            yield 拆文分析器.建立句物件(羅)
            return
        except 解析錯誤 as 錯誤:
            print(錯誤)
<|endoftext|>


<commit_before>import os.path
import webbrowser
import re

try:
    from . import api, msg, utils, reactor, shared as G
    from .handlers import account, credentials
    from .. import editor
    from ..common.exc_fmt import str_e
except (ImportError, ValueError):
    from floo.common.exc_fmt import str_e
    from floo.common.handlers import account, credentials
    from floo.common import api, msg, utils, reactor, shared as G
    from floo import editor

class FlooUI(object):
    agent = None

    def _make_agent(self, owner, workspace, auth, created_workspace, d):
        """@returns new Agent()"""
        raise NotImplemented()

    def user_y_or_n(self, context, prompt, affirmation_txt, cb):
        """@returns True/False"""
        raise NotImplemented()

    def user_select(self, context, prompt, choices_big, choices_small, cb):
        """@returns (choice, index)"""
        raise NotImplemented()

    def user_charfield(self, context, prompt, initial, cb):
        """@returns String"""
        raise NotImplemented()

    def get_a_window(self, abs_path, cb):
        """opens a project in a window or something"""
        raise NotImplemented()

    @utils.inlined_callbacks
    def link_account(self, context, host, cb):
        prompt = 'No credentials found in ~/.floorc.json for %s.  Would you like to download them (opens a browser)?.' % host
        yes = yield self.user_y_or_n, context,  prompt, "Download"
        if not yes:
            return

        agent = credentials.RequestCredentialsHandler()
        if not agent:
            self.error_message('''A configuration error occured earlier. Please go to %s and sign up to use this plugin.\n
    We're really sorry. This should never happen.''' % host)
            return

        agent.once('end', cb)

        try:
            reactor.reactor.connect(agent, host, G.DEFAULT_PORT, True)
        except Exception as e:
            print(str_e(e))

    @utils.inlined_callbacks
    def create_or_link_account(self, context, host, force, cb):
        disable_account_creation = utils.get_persistent_data().get('disable_account_creation')

        if host != "floobits.com":
            self.link_account(context, host, cb)
            return

        if disable_account_creation and not force:
            print('We could not automatically create or link your floobits account. Please go to floobits.com and sign up to use this plugin.')
            return

        if not G.EXPERT_MODE:
            editor.message_dialog('Thank you for installing the Floobits plugin!\n\nLet\'s set up your editor to work with Floobits.')

        choices = [
            'Use an existing Floobits account',
            'Create a new Floobits account',
            'Cancel (see https://floobits.com/help/floorc)'
        ]

        (choice, index) = yield self.user_select, context, 'You need a Floobits account to use Floobits! Do you want to:', choices, None

        if index == -1 or index == 2:
            d = utils.get_persistent_data()
            if not d.get('disable_account_creation'):
                d['disable_account_creation'] = True
                utils.update_persistent_data(d)
                editor.message_dialog('''You can set up a Floobits account at any time under\n\nTools -> Floobits -> Setup''')
            cb(None)
            return

        agent = None
        if index == 0:
            agent = credentials.RequestCredentialsHandler()
        else:
            agent = account.CreateAccountHandler()

        agent.once('end', cb)

        try:
            reactor.reactor.connect(agent, host, G.DEFAULT_PORT, True)
        except Exception as e:
            print(str_e(e))

    @utils.inlined_callbacks
    def remote_connect(self, context, host, owner, workspace, d, get_bufs=False):
        G.PROJECT_PATH = os.path.realpath(d)
        try:
            utils.mkdir(os.path.dirname(G.PROJECT_PATH))
        except Exception as e:
            msg.error("Couldn't create directory", G.PROJECT_PATH, str_e(e))
            return

        auth = G.AUTH.get(host)
        if not auth:
            success = yield self.link_account, context, host
            if not success:
                return
            auth = G.AUTH.get(host)
            if not auth:
                msg.error("Something went really wrong.")
                return
        if self.agent:
            try:
                self.agent.stop()
            except:
                pass

        G.WORKSPACE_WINDOW = yield self.get_a_window, d
        self.agent = self._make_agent(owner, workspace, auth, get_bufs, d)
        reactor.reactor.connect(self.agent, host, G.DEFAULT_PORT, True)
        url = self.agent.workspace_url
        utils.add_workspace_to_persistent_json(owner, workspace, url, d)
        utils.update_recent_workspaces(url)

    @utils.inlined_callbacks
    def create_workspace(self, context, host, owner, name, perms, dir_to_share, cb):
        prompt = 'Workspace name: '

        api_args = {
            'name': name,
            'owner': owner,
        }

        if perms:
            api_args['perms'] = perms

        while True:
            new_name = yield self.user_charfield, context, prompt, name
            name = new_name or name
            try:
                api_args['name'] = name
                r = api.create_workspace(host, api_args)
            except Exception as e:
                msg.error('Unable to create workspace', str_e(e))
                editor.error_message('Unable to create workspace: %s' % str_e(e))
                return

            if r.code < 400:
                workspace_url = 'https://%s/%s/%s' % (host, owner, name)
                msg.log('Created workspace', workspace_url)
                self.remote_connect(context, host, owner, name, dir_to_share, True)
                return

            msg.error('Unable to create workspace', r.body)

            if r.code not in (400, 402, 409):
                try:
                    r.body = r.body['detail']
                except Exception:
                    pass
                editor.error_message('Unable to create workspace: %s' % r.body)
                return

            if r.code == 402:
                try:
                    r.body = r.body['detail']
                except Exception:
                    pass

                yes = yield self.user_y_or_n, context, '%s Open billing settings?' % r.body, "Yes"
                if yes:
                    webbrowser.open('https://%s/%s/settings#billing' % (host, owner))
                return

            if r.code == 400:
                name = re.sub('[^A-Za-z0-9_\-\.]', '-', name)
                prompt = 'Invalid name. Workspace names must match the regex [A-Za-z0-9_\-\.]. Choose another name: '
                continue

            prompt = 'Workspace %s/%s already exists. Choose another name: ' % (owner, name)

    def join_workspace_by_url(self, context, workspace_url, possible_dirs=None):
        try:
            d = utils.parse_url(workspace_url)
        except Exception as e:
            return editor.error_message(str_e(e))

        return self.join_workspace(context, d['host'], d['workspace'], d['owner'], possible_dirs)

    @utils.inlined_callbacks
    def join_workspace(self, context, host, name, owner, possible_dirs=None):
        utils.reload_settings()

        if not utils.can_auth():
            success = yield self.create_or_link_account, context, host, False
            if not success:
                return
            utils.reload_settings()

        possible_dirs = possible_dirs or []
        for d in possible_dirs:
            info = utils.read_floo_file(d)
            if not info:
                continue
            try:
                parsed_url = utils.parse_url(info['url'])
            except Exception:
                parsed_url = None

            if parsed_url and parsed_url['host'] == host and parsed_url['workspace'] == name and parsed_url['owner'] == owner:
                self.remote_connect(context, host, owner, name, d)
                return

        try:
            d = utils.get_persistent_data()['workspaces'][owner][name]['path']
        except Exception:
            d = ''

        if os.path.isdir(d):
            self.remote_connect(context, host, owner, name, d)
            return

        d = d or os.path.join(G.SHARE_DIR or G.BASE_DIR, owner, name)
        while True:
            d = yield self.user_charfield, context, 'Save workspace files to: ', d
            if not d:
                return
            d = os.path.realpath(os.path.expanduser(d))
            if not os.path.isdir(d):
                y_or_n = yield self.user_y_or_n, context, '%s is not a directory. Create it? ' % d, "Create Directory"
                if not y_or_n:
                    return
                utils.mkdir(d)
                if not os.path.isdir(d):
                    msg.error("Couldn't create directory", d)
                    continue
            if os.path.isdir(d):
                self.remote_connect(context, host, owner, name, d)
                return

    @utils.inlined_callbacks
    def share_dir(self, context, dir_to_share, ask_about_dir, perms):
        utils.reload_settings()
        if not utils.can_auth():
            success = yield self.create_or_link_account, context, G.DEFAULT_HOST, False
            if not success:
                return
            utils.reload_settings()

        if not dir_to_share:
            if not ask_about_dir:
                raise ValueError("I need a thing to share.")
            dir_to_share = yield self.user_charfield, 'Directory to share:', ask_about_dir

        if not dir_to_share:
            return

        dir_to_share = os.path.expanduser(dir_to_share)
        dir_to_share = os.path.realpath(dir_to_share)
        dir_to_share = utils.unfuck_path(dir_to_share)

        if os.path.isfile(dir_to_share):
            dir_to_share = os.path.dirname(dir_to_share)

        workspace_name = os.path.basename(dir_to_share)
        msg.debug('', workspace_name, dir_to_share)

        if os.path.isfile(dir_to_share):
            # file_to_share = dir_to_share
            dir_to_share = os.path.dirname(dir_to_share)

        try:
            utils.mkdir(dir_to_share)
        except Exception:
            msg.error("The directory", dir_to_share, "doesn't exist and I can't create it.")
            return

        info = utils.read_floo_file(dir_to_share)

        workspace_url = info.get('url')
        if workspace_url:
            try:
                parsed_url = api.prejoin_workspace(workspace_url, dir_to_share, {'perms': perms})
            except ValueError as e:
                pass
            if parsed_url:
                # TODO: make sure we create_flooignore
                # utils.add_workspace_to_persistent_json(parsed_url['owner'], parsed_url['workspace'], workspace_url, dir_to_share)
                self.remote_connect(context, parsed_url['host'], parsed_url['owner'], parsed_url['workspace'], dir_to_share)
                return

        def prejoin(workspace_url):
            try:
                return api.prejoin_workspace(workspace_url, dir_to_share, {'perms': perms})
            except ValueError:
                pass

        parsed_url = utils.get_workspace_by_path(dir_to_share, prejoin)
        if parsed_url:
            self.remote_connect(context, parsed_url['host'], parsed_url['owner'], parsed_url['workspace'], dir_to_share)
            return

        if not G.AUTH:
            msg.warn('no auth')
            return

        hosts = list(G.AUTH.keys())
        if len(hosts) == 1:
            host = hosts[0]
        else:
            (host, index) = yield self.user_select, context, 'Which Floobits account should be used?', hosts, None
            if not host:
                return

        try:
            r = api.get_orgs_can_admin(host)
        except IOError as e:
            editor.error_message('Error getting org list: %s' % str_e(e))
            return

        # orgs = [[org['name'], 'Create workspace owned by %s' % org['name']] for org in r.body]
        # orgs.insert(0, [username, 'Create workspace owned by %s' % username])

        choices = [G.AUTH[host]['username']]
        if r.code >= 400:
            editor.error_message('Error getting org list: %s' % r.body)
        elif r.body:
            choices += [org['name'] for org in r.body]

        if len(choices) == 1:
            owner = choices[0]
        else:
            owner = yield self.user_select, context, 'Create workspace owned by', choices, None

        yield self.create_workspace, host, owner, workspace_name, perms, dir_to_share
<commit_msg>interface problems<commit_after>import os.path
import webbrowser
import re

try:
    from . import api, msg, utils, reactor, shared as G
    from .handlers import account, credentials
    from .. import editor
    from ..common.exc_fmt import str_e
except (ImportError, ValueError):
    from floo.common.exc_fmt import str_e
    from floo.common.handlers import account, credentials
    from floo.common import api, msg, utils, reactor, shared as G
    from floo import editor

class FlooUI(object):
    agent = None

    def _make_agent(self, owner, workspace, auth, created_workspace, d):
        """@returns new Agent()"""
        raise NotImplemented()

    def user_y_or_n(self, context, prompt, affirmation_txt, cb):
        """@returns True/False"""
        raise NotImplemented()

    def user_select(self, context, prompt, choices_big, choices_small, cb):
        """@returns (choice, index)"""
        raise NotImplemented()

    def user_charfield(self, context, prompt, initial, cb):
        """@returns String"""
        raise NotImplemented()

    def get_a_window(self, abs_path, cb):
        """opens a project in a window or something"""
        raise NotImplemented()

    @utils.inlined_callbacks
    def link_account(self, context, host, cb):
        prompt = 'No credentials found in ~/.floorc.json for %s.  Would you like to download them (opens a browser)?.' % host
        yes = yield self.user_y_or_n, context,  prompt, "Download"
        if not yes:
            return

        agent = credentials.RequestCredentialsHandler()
        if not agent:
            self.error_message('''A configuration error occured earlier. Please go to %s and sign up to use this plugin.\n
    We're really sorry. This should never happen.''' % host)
            return

        agent.once('end', cb)

        try:
            reactor.reactor.connect(agent, host, G.DEFAULT_PORT, True)
        except Exception as e:
            print(str_e(e))

    @utils.inlined_callbacks
    def create_or_link_account(self, context, host, force, cb):
        disable_account_creation = utils.get_persistent_data().get('disable_account_creation')

        if host != "floobits.com":
            self.link_account(context, host, cb)
            return

        if disable_account_creation and not force:
            print('We could not automatically create or link your floobits account. Please go to floobits.com and sign up to use this plugin.')
            return

        if not G.EXPERT_MODE:
            editor.message_dialog('Thank you for installing the Floobits plugin!\n\nLet\'s set up your editor to work with Floobits.')

        choices = [
            'Use an existing Floobits account',
            'Create a new Floobits account',
            'Cancel (see https://floobits.com/help/floorc)'
        ]

        (choice, index) = yield self.user_select, context, 'You need a Floobits account to use Floobits! Do you want to:', choices, None

        if index == -1 or index == 2:
            d = utils.get_persistent_data()
            if not d.get('disable_account_creation'):
                d['disable_account_creation'] = True
                utils.update_persistent_data(d)
                editor.message_dialog('''You can set up a Floobits account at any time under\n\nTools -> Floobits -> Setup''')
            cb(None)
            return

        agent = None
        if index == 0:
            agent = credentials.RequestCredentialsHandler()
        else:
            agent = account.CreateAccountHandler()

        agent.once('end', cb)

        try:
            reactor.reactor.connect(agent, host, G.DEFAULT_PORT, True)
        except Exception as e:
            print(str_e(e))

    @utils.inlined_callbacks
    def remote_connect(self, context, host, owner, workspace, d, get_bufs=False):
        G.PROJECT_PATH = os.path.realpath(d)
        try:
            utils.mkdir(os.path.dirname(G.PROJECT_PATH))
        except Exception as e:
            msg.error("Couldn't create directory", G.PROJECT_PATH, str_e(e))
            return

        auth = G.AUTH.get(host)
        if not auth:
            success = yield self.link_account, context, host
            if not success:
                return
            auth = G.AUTH.get(host)
            if not auth:
                msg.error("Something went really wrong.")
                return
        if self.agent:
            try:
                self.agent.stop()
            except:
                pass

        G.WORKSPACE_WINDOW = yield self.get_a_window, d
        self.agent = self._make_agent(owner, workspace, auth, get_bufs, d)
        reactor.reactor.connect(self.agent, host, G.DEFAULT_PORT, True)
        url = self.agent.workspace_url
        utils.add_workspace_to_persistent_json(owner, workspace, url, d)
        utils.update_recent_workspaces(url)

    @utils.inlined_callbacks
    def create_workspace(self, context, host, owner, name, perms, dir_to_share, cb):
        prompt = 'Workspace name: '

        api_args = {
            'name': name,
            'owner': owner,
        }

        if perms:
            api_args['perms'] = perms

        while True:
            new_name = yield self.user_charfield, context, prompt, name
            name = new_name or name
            try:
                api_args['name'] = name
                r = api.create_workspace(host, api_args)
            except Exception as e:
                msg.error('Unable to create workspace ', str_e(e))
                editor.error_message('Unable to create workspace: %s' % str_e(e))
                return

            if r.code < 400:
                workspace_url = 'https://%s/%s/%s' % (host, owner, name)
                msg.log('Created workspace', workspace_url)
                self.remote_connect(context, host, owner, name, dir_to_share, True)
                return

            msg.error('Unable to create workspace', r.body)

            if r.code not in (400, 402, 409):
                try:
                    r.body = r.body['detail']
                except Exception:
                    pass
                editor.error_message('Unable to create workspace: %s' % r.body)
                return

            if r.code == 402:
                try:
                    r.body = r.body['detail']
                except Exception:
                    pass

                yes = yield self.user_y_or_n, context, '%s Open billing settings?' % r.body, "Yes"
                if yes:
                    webbrowser.open('https://%s/%s/settings#billing' % (host, owner))
                return

            if r.code == 400:
                name = re.sub('[^A-Za-z0-9_\-\.]', '-', name)
                prompt = 'Invalid name. Workspace names must match the regex [A-Za-z0-9_\-\.]. Choose another name: '
                continue

            prompt = 'Workspace %s/%s already exists. Choose another name: ' % (owner, name)

    def join_workspace_by_url(self, context, workspace_url, possible_dirs=None):
        try:
            d = utils.parse_url(workspace_url)
        except Exception as e:
            return editor.error_message(str_e(e))

        return self.join_workspace(context, d['host'], d['workspace'], d['owner'], possible_dirs)

    @utils.inlined_callbacks
    def join_workspace(self, context, host, name, owner, possible_dirs=None):
        utils.reload_settings()

        if not utils.can_auth():
            success = yield self.create_or_link_account, context, host, False
            if not success:
                return
            utils.reload_settings()

        possible_dirs = possible_dirs or []
        for d in possible_dirs:
            info = utils.read_floo_file(d)
            if not info:
                continue
            try:
                parsed_url = utils.parse_url(info['url'])
            except Exception:
                parsed_url = None

            if parsed_url and parsed_url['host'] == host and parsed_url['workspace'] == name and parsed_url['owner'] == owner:
                self.remote_connect(context, host, owner, name, d)
                return

        try:
            d = utils.get_persistent_data()['workspaces'][owner][name]['path']
        except Exception:
            d = ''

        if d and os.path.isdir(d):
            self.remote_connect(context, host, owner, name, d)
            return

        d = d or os.path.join(G.SHARE_DIR or G.BASE_DIR, owner, name)
        while True:
            d = yield self.user_charfield, context, 'Save workspace files to: ', d
            if not d:
                return
            d = os.path.realpath(os.path.expanduser(d))
            if not os.path.isdir(d):
                y_or_n = yield self.user_y_or_n, context, '%s is not a directory. Create it? ' % d, "Create Directory"
                if not y_or_n:
                    return
                utils.mkdir(d)
                if not os.path.isdir(d):
                    msg.error("Couldn't create directory", d)
                    continue
            if os.path.isdir(d):
                self.remote_connect(context, host, owner, name, d)
                return

    @utils.inlined_callbacks
    def share_dir(self, context, dir_to_share, ask_about_dir, perms):
        utils.reload_settings()
        if not utils.can_auth():
            success = yield self.create_or_link_account, context, G.DEFAULT_HOST, False
            if not success:
                return
            utils.reload_settings()

        if not dir_to_share:
            if not ask_about_dir:
                raise ValueError("I need a thing to share.")
            dir_to_share = yield self.user_charfield, 'Directory to share:', ask_about_dir

        if not dir_to_share:
            return

        dir_to_share = os.path.expanduser(dir_to_share)
        dir_to_share = os.path.realpath(dir_to_share)
        dir_to_share = utils.unfuck_path(dir_to_share)

        if os.path.isfile(dir_to_share):
            dir_to_share = os.path.dirname(dir_to_share)

        workspace_name = os.path.basename(dir_to_share)
        msg.debug('', workspace_name, dir_to_share)

        if os.path.isfile(dir_to_share):
            # file_to_share = dir_to_share
            dir_to_share = os.path.dirname(dir_to_share)

        try:
            utils.mkdir(dir_to_share)
        except Exception:
            msg.error("The directory", dir_to_share, "doesn't exist and I can't create it.")
            return

        info = utils.read_floo_file(dir_to_share)

        workspace_url = info.get('url')
        if workspace_url:
            try:
                parsed_url = api.prejoin_workspace(workspace_url, dir_to_share, {'perms': perms})
            except ValueError as e:
                pass
            if parsed_url:
                # TODO: make sure we create_flooignore
                self.remote_connect(context, parsed_url['host'], parsed_url['owner'], parsed_url['workspace'], dir_to_share)
                return

        def prejoin(workspace_url):
            try:
                return api.prejoin_workspace(workspace_url, dir_to_share, {'perms': perms})
            except ValueError:
                pass

        parsed_url = utils.get_workspace_by_path(dir_to_share, prejoin)
        if parsed_url:
            self.remote_connect(context, parsed_url['host'], parsed_url['owner'], parsed_url['workspace'], dir_to_share)
            return

        if not G.AUTH:
            msg.warn('no auth')
            return

        hosts = list(G.AUTH.keys())
        if len(hosts) == 1:
            host = hosts[0]
        else:
            (host, index) = yield self.user_select, context, 'Which Floobits account should be used?', hosts, None
            if not host:
                return

        try:
            r = api.get_orgs_can_admin(host)
        except IOError as e:
            editor.error_message('Error getting org list: %s' % str_e(e))
            return

        # orgs = [[org['name'], 'Create workspace owned by %s' % org['name']] for org in r.body]
        # orgs.insert(0, [username, 'Create workspace owned by %s' % username])

        choices = [G.AUTH[host]['username']]
        if r.code >= 400:
            editor.error_message('Error getting org list: %s' % r.body)
        elif r.body:
            choices += [org['name'] for org in r.body]

        if len(choices) == 1:
            owner = choices[0]
        else:
            (owner, index) = yield self.user_select, context, 'Create workspace owned by', choices, None

        yield self.create_workspace, context, host, owner, workspace_name, perms, dir_to_share
<|endoftext|>


<commit_before>from django import forms
from django.utils.translation import ugettext_lazy as _

from .widgets import (TextInput, HiddenInput, CheckboxInput, Select,
                      ClearableFileInput, SelectMultiple, DateInput,
                      DateTimeInput, TimeInput, URLInput, NumberInput,
                      EmailInput, NullBooleanSelect, SlugInput, IPAddressInput,
                      SplitDateTimeWidget, SplitHiddenDateTimeWidget,
                      FILE_INPUT_CONTRADICTION)

__all__ = (
    'Field', 'CharField', 'IntegerField', 'DateField', 'TimeField',
    'DateTimeField', 'EmailField', 'FileField', 'ImageField', 'URLField',
    'BooleanField', 'NullBooleanField', 'ChoiceField', 'MultipleChoiceField',
    'FloatField', 'DecimalField', 'SlugField', 'RegexField', 'IPAddressField',
    'TypedChoiceField', 'FilePathField', 'TypedMultipleChoiceField',
    'ComboField', 'MultiValueField', 'SplitDateTimeField',
)

class Field(forms.Field):
    widget = TextInput
    hidden_widget = HiddenInput

    def __init__(self, *args, **kwargs):
        super(Field, self).__init__(*args, **kwargs)
        self.widget.is_required = self.required  # fallback to support
                                                 # is_required with
                                                 # Django < 1.3

class CharField(Field, forms.CharField):
    widget = TextInput

    def widget_attrs(self, widget):
        attrs = super(CharField, self).widget_attrs(widget)
        if self.max_length is not None and isinstance(widget, (TextInput, HiddenInput)):
            # The HTML attribute is maxlength, not max_length.
            attrs.update({'maxlength': str(self.max_length)})
        return attrs

class BooleanField(Field, forms.BooleanField):
    widget = CheckboxInput

class NullBooleanField(Field, forms.NullBooleanField):
    widget = NullBooleanSelect

class ChoiceField(Field, forms.ChoiceField):
    widget = Select

class TypedChoiceField(ChoiceField, forms.TypedChoiceField):
    widget = Select

class FilePathField(ChoiceField, forms.FilePathField):
    widget = Select

class FileField(Field, forms.FileField):
    widget = ClearableFileInput
    default_error_messages = {
        'invalid': _(u"No file was submitted. Check the encoding type on the form."),
        'missing': _(u"No file was submitted."),
        'empty': _(u"The submitted file is empty."),
        'max_length': _(u'Ensure this filename has at most %(max)d characters (it has %(length)d).'),
        'contradiction': _(u'Please either submit a file or check the clear checkbox, not both.')
    }

    def __init__(self, *args, **kwargs):
        self.max_length = kwargs.pop('max_length', None)
        self.allow_empty_file = kwargs.pop('allow_empty_file', False)
        super(FileField, self).__init__(*args, **kwargs)

    def clean(self, data, initial=None):
        if data is FILE_INPUT_CONTRADICTION:
            raise ValidationError(self.error_messages['contradiction'])
        if data is False:
            if not self.required:
                return False
            data = None
        if not data and initial:
            return initial
        return super(FileField, self).clean(data)

class ImageField(Field, forms.ImageField):
    widget = ClearableFileInput

class MultipleChoiceField(Field, forms.MultipleChoiceField):
    widget = SelectMultiple

try:
    Parent = forms.TypedMultipleChoiceField
except AttributeError:  # Django < 1.3
    class Parent(forms.MultipleChoiceField):
        """No-op class for older Django versions"""
        def __init__(self, *args, **kwargs):
            kwargs.pop('coerce', None)
            kwargs.pop('empty_value', None)
            super(Parent, self).__init__(*args, **kwargs)

class TypedMultipleChoiceField(MultipleChoiceField, Parent):
    pass

class DateField(Field, forms.DateField):
    widget = DateInput

class DateTimeField(Field, forms.DateTimeField):
    widget = DateTimeInput

class TimeField(Field, forms.TimeField):
    widget = TimeInput

class DecimalField(Field, forms.DecimalField):
    widget = NumberInput

class FloatField(Field, forms.FloatField):
    widget = NumberInput

class IntegerField(FloatField, forms.IntegerField):
    widget = NumberInput

class EmailField(Field, forms.EmailField):
    widget = EmailInput

class URLField(Field, forms.URLField):
    widget = URLInput

class SlugField(Field, forms.SlugField):
    widget = SlugInput

class RegexField(Field, forms.RegexField):
    widget = TextInput

    def __init__(self, regex, js_regex=None, max_length=None, min_length=None,
                 error_message=None, *args, **kwargs):
        self.js_regex = js_regex
        super(RegexField, self).__init__(regex, max_length, min_length,
                                         *args, **kwargs)

    def widget_attrs(self, widget):
        attrs = super(RegexField, self).widget_attrs(widget) or {}
        if self.js_regex is not None:
            attrs['pattern'] = self.js_regex
        return attrs

class IPAddressField(Field, forms.IPAddressField):
    widget = IPAddressInput

class ComboField(Field, forms.ComboField):
    pass

class MultiValueField(Field, forms.MultiValueField):
    pass

class SplitDateTimeField(forms.SplitDateTimeField):
    widget = SplitDateTimeWidget
    hidden_widget = SplitHiddenDateTimeWidget

    def __init__(self, *args, **kwargs):
        super(SplitDateTimeField, self).__init__(*args, **kwargs)
        for widget in self.widget.widgets:
            widget.is_required = self.required
<commit_msg>Compatibility fix for Django 1.3.<commit_after>from django import forms
from django.utils.translation import ugettext_lazy as _

from .widgets import (TextInput, HiddenInput, CheckboxInput, Select,
                      ClearableFileInput, SelectMultiple, DateInput,
                      DateTimeInput, TimeInput, URLInput, NumberInput,
                      EmailInput, NullBooleanSelect, SlugInput, IPAddressInput,
                      SplitDateTimeWidget, SplitHiddenDateTimeWidget,
                      FILE_INPUT_CONTRADICTION)

__all__ = (
    'Field', 'CharField', 'IntegerField', 'DateField', 'TimeField',
    'DateTimeField', 'EmailField', 'FileField', 'ImageField', 'URLField',
    'BooleanField', 'NullBooleanField', 'ChoiceField', 'MultipleChoiceField',
    'FloatField', 'DecimalField', 'SlugField', 'RegexField', 'IPAddressField',
    'TypedChoiceField', 'FilePathField', 'TypedMultipleChoiceField',
    'ComboField', 'MultiValueField', 'SplitDateTimeField',
)

class Field(forms.Field):
    widget = TextInput
    hidden_widget = HiddenInput

    def __init__(self, *args, **kwargs):
        super(Field, self).__init__(*args, **kwargs)
        self.widget.is_required = self.required  # fallback to support
                                                 # is_required with
                                                 # Django < 1.3

class CharField(Field, forms.CharField):
    widget = TextInput

    def widget_attrs(self, widget):
        attrs = super(CharField, self).widget_attrs(widget)
        if attrs is None:
            attrs = {}
        if self.max_length is not None and isinstance(widget, (TextInput, HiddenInput)):
            # The HTML attribute is maxlength, not max_length.
            attrs.update({'maxlength': str(self.max_length)})
        return attrs

class BooleanField(Field, forms.BooleanField):
    widget = CheckboxInput

class NullBooleanField(Field, forms.NullBooleanField):
    widget = NullBooleanSelect

class ChoiceField(Field, forms.ChoiceField):
    widget = Select

class TypedChoiceField(ChoiceField, forms.TypedChoiceField):
    widget = Select

class FilePathField(ChoiceField, forms.FilePathField):
    widget = Select

class FileField(Field, forms.FileField):
    widget = ClearableFileInput
    default_error_messages = {
        'invalid': _(u"No file was submitted. Check the encoding type on the form."),
        'missing': _(u"No file was submitted."),
        'empty': _(u"The submitted file is empty."),
        'max_length': _(u'Ensure this filename has at most %(max)d characters (it has %(length)d).'),
        'contradiction': _(u'Please either submit a file or check the clear checkbox, not both.')
    }

    def __init__(self, *args, **kwargs):
        self.max_length = kwargs.pop('max_length', None)
        self.allow_empty_file = kwargs.pop('allow_empty_file', False)
        super(FileField, self).__init__(*args, **kwargs)

    def clean(self, data, initial=None):
        if data is FILE_INPUT_CONTRADICTION:
            raise ValidationError(self.error_messages['contradiction'])
        if data is False:
            if not self.required:
                return False
            data = None
        if not data and initial:
            return initial
        return super(FileField, self).clean(data)

class ImageField(Field, forms.ImageField):
    widget = ClearableFileInput

class MultipleChoiceField(Field, forms.MultipleChoiceField):
    widget = SelectMultiple

try:
    Parent = forms.TypedMultipleChoiceField
except AttributeError:  # Django < 1.3
    class Parent(forms.MultipleChoiceField):
        """No-op class for older Django versions"""
        def __init__(self, *args, **kwargs):
            kwargs.pop('coerce', None)
            kwargs.pop('empty_value', None)
            super(Parent, self).__init__(*args, **kwargs)

class TypedMultipleChoiceField(MultipleChoiceField, Parent):
    pass

class DateField(Field, forms.DateField):
    widget = DateInput

class DateTimeField(Field, forms.DateTimeField):
    widget = DateTimeInput

class TimeField(Field, forms.TimeField):
    widget = TimeInput

class DecimalField(Field, forms.DecimalField):
    widget = NumberInput

class FloatField(Field, forms.FloatField):
    widget = NumberInput

class IntegerField(FloatField, forms.IntegerField):
    widget = NumberInput

class EmailField(Field, forms.EmailField):
    widget = EmailInput

class URLField(Field, forms.URLField):
    widget = URLInput

class SlugField(Field, forms.SlugField):
    widget = SlugInput

class RegexField(Field, forms.RegexField):
    widget = TextInput

    def __init__(self, regex, js_regex=None, max_length=None, min_length=None,
                 error_message=None, *args, **kwargs):
        self.js_regex = js_regex
        super(RegexField, self).__init__(regex, max_length, min_length,
                                         *args, **kwargs)

    def widget_attrs(self, widget):
        attrs = super(RegexField, self).widget_attrs(widget) or {}
        if self.js_regex is not None:
            attrs['pattern'] = self.js_regex
        return attrs

class IPAddressField(Field, forms.IPAddressField):
    widget = IPAddressInput

class ComboField(Field, forms.ComboField):
    pass

class MultiValueField(Field, forms.MultiValueField):
    pass

class SplitDateTimeField(forms.SplitDateTimeField):
    widget = SplitDateTimeWidget
    hidden_widget = SplitHiddenDateTimeWidget

    def __init__(self, *args, **kwargs):
        super(SplitDateTimeField, self).__init__(*args, **kwargs)
        for widget in self.widget.widgets:
            widget.is_required = self.required
<|endoftext|>


<commit_before>import os
import logging

import itertools

import tabulator

from jsontableschema import Schema

from datapackage_pipelines.wrapper import ingest, spew
from datapackage_pipelines.utilities.resource_matcher import ResourceMatcher
from datapackage_pipelines.utilities.tabulator_txt_parser import TXTParser

def _reader(opener, _url):
    yield None
    filename = os.path.basename(_url)
    logging.info('%s: OPENING %s', filename, _url)
    _schema, _headers, _reader, _close = opener()
    num_headers = len(_headers)
    i = 0
    for i, row in enumerate(_reader):
        row = [str(x).strip() for x in row]
        values = set(row)
        if len(values) == 1 and '' in values:
            # In case of empty rows, just skip them
            continue
        output = dict(zip(_headers, _schema.cast_row(row[:num_headers])))
        yield output

        i += 1
        if i % 10000 == 0:
            logging.info('%s: %d rows', filename, i)
            # break
    _close()
    logging.info('%s: TOTAL %d rows', filename, i)

def dedupe(headers):
    _dedupped_headers = []
    headers = list(map(str, headers))
    for hdr in headers:
        if hdr is None:
            continue
        hdr = hdr.strip()
        if len(hdr) == 0:
            continue
        if hdr in _dedupped_headers:
            i = 0
            deduped_hdr = hdr
            while deduped_hdr in _dedupped_headers:
                i += 1
                deduped_hdr = '%s_%s' % (hdr, i)
            hdr = deduped_hdr
        _dedupped_headers.append(hdr)
    return _dedupped_headers

def row_skipper(rows_to_skip):
    def _func(extended_rows):
        for number, headers, row in extended_rows:
            if number > rows_to_skip:
                yield (number-rows_to_skip, headers, row)
    return _func

def add_constants(extra_headers, extra_values):
    def _func(extended_rows):
        for number, headers, row in extended_rows:
            row.extend(extra_values)
            yield number, headers + extra_headers, row
    return _func

def suffix_remover():
    def empty_row(row):
        return all(v is None or (isinstance(v, str) and v == '')
                   for v in row)

    def _func(extended_rows):
        stage = 'prefix'
        for number, headers, row in extended_rows:
            if stage == 'suffix':
                if not empty_row(row):
                    raise ValueError('Expected an empty row, but got %r instead' % row)
            elif stage == 'body':
                if empty_row(row):
                    stage = 'suffix'
                else:
                    yield number, headers, row
            elif stage == 'prefix':
                if not empty_row(row):
                    stage = 'body'
                yield number, headers, row

    return _func

def stream_reader(_resource, _url, _ignore_missing):
    def get_opener(__url, __resource):
        def opener():
            _params = dict(headers=1)
            _params.update(
                dict(x for x in __resource.items()
                     if x[0] not in {'path', 'name', 'schema',
                                     'mediatype', 'skip_rows',
                                     'constants'}))
            skip_rows = __resource.get('skip_rows', 0)
            if _params.get("format") == "txt":
                # datapackage-pipelines processing requires having a header row
                # for txt format we add a single "data" column
                _params["headers"] = ["data"]
                _params["custom_parsers"] = {"txt": TXTParser}
                _params["allow_html"] = True
            constants = _resource.get('constants', {})
            constant_headers = list(constants.keys())
            constant_values = [constants.get(k) for k in constant_headers]
            _stream = tabulator.Stream(__url, **_params,
                                       post_parse=[row_skipper(skip_rows),
                                                   suffix_remover(),
                                                   add_constants(constant_headers, constant_values)])
            try:
                _stream.open()
                _headers = dedupe(_stream.headers + constant_headers)
                _schema = __resource.get('schema')
                if _schema is not None:
                    _schema = Schema(_schema)
                return _schema, _headers, _stream, _stream.close
            except tabulator.exceptions.TabulatorException as e:
                logging.warning("Error while opening resource from url %s: %r",
                                _url, e)
                _stream.close()
                if not _ignore_missing:
                    raise
                return {}, [], [], lambda: None
        return opener

    schema, headers, stream, close = get_opener(url, _resource)()
    if schema is None:
        schema = {
            'fields': [
                {'name': header, 'type': 'string'}
                for header in headers
                ]
        }
        _resource['schema'] = schema

    close()
    del stream

    return itertools\
        .islice(
            _reader(
                get_opener(_url, _resource),
                _url),
            1, None)

parameters, datapackage, resource_iterator = ingest()

resources = ResourceMatcher(parameters.get('resources'))
ignore_missing = parameters.get('ignore-missing', False)

new_resource_iterator = []
for resource in datapackage['resources']:

    path = resource.get('path')
    if path is not None and '://' not in path:
        new_resource_iterator.append(next(resource_iterator))
    elif 'url' in resource:
        url = resource['url']

        name = resource['name']
        if not resources.match(name):
            continue

        path = os.path.join('data', name + '.csv')
        resource['path'] = path

        del resource['url']
        rows = stream_reader(resource, url, ignore_missing or url == "")
        new_resource_iterator.append(rows)

spew(datapackage, new_resource_iterator)
<commit_msg>Special exemption for txt format in suffix remover<commit_after>import os
import logging

import itertools

import tabulator

from jsontableschema import Schema

from datapackage_pipelines.wrapper import ingest, spew
from datapackage_pipelines.utilities.resource_matcher import ResourceMatcher
from datapackage_pipelines.utilities.tabulator_txt_parser import TXTParser

def _reader(opener, _url):
    yield None
    filename = os.path.basename(_url)
    logging.info('%s: OPENING %s', filename, _url)
    _schema, _headers, _reader, _close = opener()
    num_headers = len(_headers)
    i = 0
    for i, row in enumerate(_reader):
        row = [str(x).strip() for x in row]
        values = set(row)
        if len(values) == 1 and '' in values:
            # In case of empty rows, just skip them
            continue
        output = dict(zip(_headers, _schema.cast_row(row[:num_headers])))
        yield output

        i += 1
        if i % 10000 == 0:
            logging.info('%s: %d rows', filename, i)
            # break
    _close()
    logging.info('%s: TOTAL %d rows', filename, i)

def dedupe(headers):
    _dedupped_headers = []
    headers = list(map(str, headers))
    for hdr in headers:
        if hdr is None:
            continue
        hdr = hdr.strip()
        if len(hdr) == 0:
            continue
        if hdr in _dedupped_headers:
            i = 0
            deduped_hdr = hdr
            while deduped_hdr in _dedupped_headers:
                i += 1
                deduped_hdr = '%s_%s' % (hdr, i)
            hdr = deduped_hdr
        _dedupped_headers.append(hdr)
    return _dedupped_headers

def row_skipper(rows_to_skip):
    def _func(extended_rows):
        for number, headers, row in extended_rows:
            if number > rows_to_skip:
                yield (number-rows_to_skip, headers, row)
    return _func

def add_constants(extra_headers, extra_values):
    def _func(extended_rows):
        for number, headers, row in extended_rows:
            row.extend(extra_values)
            yield number, headers + extra_headers, row
    return _func

def suffix_remover(format):
    def empty_row(row):
        return all(v is None or (isinstance(v, str) and v == '')
                   for v in row)

    def _func(extended_rows):
        stage = 'prefix'
        for number, headers, row in extended_rows:
            if format == 'txt':
                yield number, headers, row
                continue

            if stage == 'suffix':
                if not empty_row(row):
                    logging.warning('Expected an empty row, but got %r instead' % row)
            elif stage == 'body':
                if empty_row(row):
                    stage = 'suffix'
                else:
                    yield number, headers, row
            elif stage == 'prefix':
                if not empty_row(row):
                    stage = 'body'
                yield number, headers, row

    return _func

def stream_reader(_resource, _url, _ignore_missing):
    def get_opener(__url, __resource):
        def opener():
            _params = dict(headers=1)
            _params.update(
                dict(x for x in __resource.items()
                     if x[0] not in {'path', 'name', 'schema',
                                     'mediatype', 'skip_rows',
                                     'constants'}))
            skip_rows = __resource.get('skip_rows', 0)
            format = _params.get("format")
            if format == "txt":
                # datapackage-pipelines processing requires having a header row
                # for txt format we add a single "data" column
                _params["headers"] = ["data"]
                _params["custom_parsers"] = {"txt": TXTParser}
                _params["allow_html"] = True
            constants = _resource.get('constants', {})
            constant_headers = list(constants.keys())
            constant_values = [constants.get(k) for k in constant_headers]
            _stream = tabulator.Stream(__url, **_params,
                                       post_parse=[row_skipper(skip_rows),
                                                   suffix_remover(format),
                                                   add_constants(constant_headers, constant_values)])
            try:
                _stream.open()
                _headers = dedupe(_stream.headers + constant_headers)
                _schema = __resource.get('schema')
                if _schema is not None:
                    _schema = Schema(_schema)
                return _schema, _headers, _stream, _stream.close
            except tabulator.exceptions.TabulatorException as e:
                logging.warning("Error while opening resource from url %s: %r",
                                _url, e)
                _stream.close()
                if not _ignore_missing:
                    raise
                return {}, [], [], lambda: None
        return opener

    schema, headers, stream, close = get_opener(url, _resource)()
    if schema is None:
        schema = {
            'fields': [
                {'name': header, 'type': 'string'}
                for header in headers
                ]
        }
        _resource['schema'] = schema

    close()
    del stream

    return itertools\
        .islice(
            _reader(
                get_opener(_url, _resource),
                _url),
            1, None)

parameters, datapackage, resource_iterator = ingest()

resources = ResourceMatcher(parameters.get('resources'))
ignore_missing = parameters.get('ignore-missing', False)

new_resource_iterator = []
for resource in datapackage['resources']:

    path = resource.get('path')
    if path is not None and '://' not in path:
        new_resource_iterator.append(next(resource_iterator))
    elif 'url' in resource:
        url = resource['url']

        name = resource['name']
        if not resources.match(name):
            continue

        path = os.path.join('data', name + '.csv')
        resource['path'] = path

        del resource['url']
        rows = stream_reader(resource, url, ignore_missing or url == "")
        new_resource_iterator.append(rows)

spew(datapackage, new_resource_iterator)
<|endoftext|>


<commit_before># coding=UTF8
__author__ = 'daedaluschan'

import sys, time
import telepot
from telepot.delegate import per_chat_id, create_open

from datetime import date
from types import *
from chk_n_conv import  chkNConv
from enum import Enum

to_buy_list = []
white_list = [161517202]
file_name = 'list.txt'

class ConverType(Enum):
    nothing = 1
    asking_date = 2
    adding_to_buy = 3

class Jinny:
  myName = u'Jinny Chan'
  myDOB = date(2015,6,24)

  @staticmethod
  def getNumOfDays():
    currentDate = date.today()
    return (currentDate - Jinny.myDOB).days + 1

  @staticmethod
  def getNumOfDaysSpecific(whichDateIn):
    if type(whichDateIn) is StringType or type(whichDateIn) is UnicodeType:
      whichDate = u''
      for charact in whichDateIn:
        if charact.isdigit():
          whichDate = whichDate + charact
      theDate = date(int(whichDate[0:4]),int(whichDate[4:6]),int(whichDate[6:8]))
    else:
      theDate = whichDateIn

    return (theDate - Jinny.myDOB).days + 1

class AngryJinny(telepot.helper.ChatHandler):
    def __init__(self, seed_tuple, timeout):
        super(AngryJinny, self).__init__(seed_tuple, timeout)
        self._convert_type = ConverType.nothing
        print('constructor is being called')

    def genKeyboard(self):
        print('generate keyboard')
        show_keyboard = {'keyboard': [[u'今日 day 幾',u'某日係 day 幾'], [u'有乜未買？', u'有野要買']]}

        for idx, item in to_buy_list:
            print(chkNConv(idx.__str__()) )
            show_keyboard['keyboard'].append([u'買左[' + chkNConv(idx.__str__()) + u']' + chkNConv(item)])
        return show_keyboard

    def genBuyList(self):
        buy_list = u'要買呢D：\n\n'
        for item in to_buy_list:
            buy_list = buy_list + u'- ' + chkNConv(item) + u'\n'

        return buy_list

    def doneWithBuyList(self):
        self.sender.sendMessage(text=u'Bye', reply_markup=self.genKeyboard())
        for white_user in white_list:
            self.bot.sendMessage(white_user, self.genBuyList())

    def writeListToFile(self):
        with open(name=file_name, mode='w') as f:
            for item in to_buy_list:
                f.write(item.encode('utf-8'))
                f.write(u'\n'.encode('utf-8'))
        f.close()

    def on_message(self, msg):
        print('on_message() is being called')
        flavor = telepot.flavor(msg)

        # normal message
        if flavor == 'normal':
            content_type, chat_type, _chat_id = telepot.glance2(msg)
            print('Normal Message:', content_type, chat_type, _chat_id, '; message content: ', msg)

            if _chat_id in white_list :

                if self._convert_type == ConverType.nothing:
                    if chkNConv(msg['text']) == u'/start' or chkNConv(msg['text']) == u'/today' or chkNConv(msg['text']) == u'今日 day 幾':
                        self.sender.sendMessage(text=u'Today is ' + chkNConv(str(date.today())) + u'. \n' +
                                                               u'It is my day ' + chkNConv(str(Jinny.getNumOfDays())) + u'. \n' +
                                                               u'Use /help for more options',
                                                reply_markup=self.genKeyboard())
                    elif chkNConv(msg['text']) == u'/help':
                        self.sender.sendMessage(text=u'/today - get today\'s date and my day count. \n' +
                                                               u'/help - for those who have bad memory. \n' +
                                                               u'/query - check my day count for a particular date. \n',
                                                reply_markup=self.genKeyboard())
                    elif chkNConv(msg['text']) == u'/query' or chkNConv(msg['text']) == u'某日係 day 幾':
                        self._convert_type = ConverType.asking_date
                        self.sender.sendMessage(text=u'邊日呀 ?', reply_markup={'hide_keyboard': True})
                    elif chkNConv(msg['text']) == u'/No' :
                        self._convert_type = ConverType.nothing
                        self.sender.sendMessage(text=u'Bye !',
                                                reply_markup=self.genKeyboard())
                    elif chkNConv(msg['text']) == u'有野要買':
                        self._convert_type = ConverType.adding_to_buy
                        self.sender.sendMessage(text=u'買乜？', reply_markup={'hide_keyboard': True})
                    elif chkNConv(msg['text']) == u'有乜未買？':
                        self.sender.sendMessage(text=self.genBuyList(), reply_markup=self.genKeyboard())
                    else:
                        self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                     u'你試多次啦。或者用 /help 我就教你用。')
                elif self._convert_type == ConverType.asking_date:
                    try :
                        self.sender.sendMessage(text=u'For ' + chkNConv(msg['text']) + u', It is my day ' +
                                                     chkNConv(str(Jinny.getNumOfDaysSpecific(msg['text']))) +
                                                     u'. Any other date to ask ? /No ?')
                    except BaseException :
                        self.sender.sendMessage(text=u'Not a data that I can understand ! \n' +
                                                     u'What date (YYYYMMDD) ? \nOr you done ? ( /No )')
                elif self._convert_type == ConverType.adding_to_buy:
                    if chkNConv(msg['text']) == u'/done':
                        self._convert_type = ConverType.nothing
                        self.doneWithBuyList()
                        self.writeListToFile()
                    else:
                        to_buy_list.append(chkNConv(msg['text']))
                        self.sender.sendMessage(text=u'仲有冇？如果冇，就用 /done 完結。', reply_markup={'hide_keyboard': True})
                else:
                    self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                 u'你試多次啦。或者用 /help 我就教你用。')

            else:
                self.sender.sendMessage(text=u'你邊位呀？')
        else:
            raise telepot.BadFlavor(msg)

        print(u'to buy list: ' + self.genBuyList())

TOKEN = sys.argv[1]  # get token from command-line

with open(name=file_name, mode='r') as f:
    lines = f.read().decode('utf-8').splitlines()
f.close()
to_buy_list = lines

bot = telepot.DelegatorBot(TOKEN, [
    (per_chat_id(), create_open(AngryJinny, timeout=30)),])
print('Listening ...')
bot.notifyOnMessage(run_forever=True)

<commit_msg>write list to file<commit_after># coding=UTF8
__author__ = 'daedaluschan'

import sys, time
import telepot
from telepot.delegate import per_chat_id, create_open

from datetime import date
from types import *
from chk_n_conv import  chkNConv
from enum import Enum

to_buy_list = []
white_list = [161517202]
file_name = 'list.txt'

class ConverType(Enum):
    nothing = 1
    asking_date = 2
    adding_to_buy = 3

class Jinny:
  myName = u'Jinny Chan'
  myDOB = date(2015,6,24)

  @staticmethod
  def getNumOfDays():
    currentDate = date.today()
    return (currentDate - Jinny.myDOB).days + 1

  @staticmethod
  def getNumOfDaysSpecific(whichDateIn):
    if type(whichDateIn) is StringType or type(whichDateIn) is UnicodeType:
      whichDate = u''
      for charact in whichDateIn:
        if charact.isdigit():
          whichDate = whichDate + charact
      theDate = date(int(whichDate[0:4]),int(whichDate[4:6]),int(whichDate[6:8]))
    else:
      theDate = whichDateIn

    return (theDate - Jinny.myDOB).days + 1

class AngryJinny(telepot.helper.ChatHandler):
    def __init__(self, seed_tuple, timeout):
        super(AngryJinny, self).__init__(seed_tuple, timeout)
        self._convert_type = ConverType.nothing
        print('constructor is being called')

    def genKeyboard(self):
        print('generate keyboard')
        show_keyboard = {'keyboard': [[u'今日 day 幾',u'某日係 day 幾'], [u'有乜未買？', u'有野要買']]}

        for idx, item in to_buy_list:
            print(u'買左[' + idx.__str__() + u']')
            show_keyboard['keyboard'].append([u'買左[' + chkNConv(idx.__str__()) + u']' + chkNConv(item)])
        return show_keyboard

    def genBuyList(self):
        buy_list = u'要買呢D：\n\n'
        for item in to_buy_list:
            buy_list = buy_list + u'- ' + chkNConv(item) + u'\n'

        return buy_list

    def doneWithBuyList(self):
        self.sender.sendMessage(text=u'Bye', reply_markup=self.genKeyboard())
        for white_user in white_list:
            self.bot.sendMessage(white_user, self.genBuyList())

    def writeListToFile(self):
        with open(name=file_name, mode='w') as f:
            for item in to_buy_list:
                f.write(item.encode('utf-8'))
                f.write(u'\n'.encode('utf-8'))
        f.close()

    def on_message(self, msg):
        print('on_message() is being called')
        flavor = telepot.flavor(msg)

        # normal message
        if flavor == 'normal':
            content_type, chat_type, _chat_id = telepot.glance2(msg)
            print('Normal Message:', content_type, chat_type, _chat_id, '; message content: ', msg)

            if _chat_id in white_list :

                if self._convert_type == ConverType.nothing:
                    if chkNConv(msg['text']) == u'/start' or chkNConv(msg['text']) == u'/today' or chkNConv(msg['text']) == u'今日 day 幾':
                        self.sender.sendMessage(text=u'Today is ' + chkNConv(str(date.today())) + u'. \n' +
                                                               u'It is my day ' + chkNConv(str(Jinny.getNumOfDays())) + u'. \n' +
                                                               u'Use /help for more options',
                                                reply_markup=self.genKeyboard())
                    elif chkNConv(msg['text']) == u'/help':
                        self.sender.sendMessage(text=u'/today - get today\'s date and my day count. \n' +
                                                               u'/help - for those who have bad memory. \n' +
                                                               u'/query - check my day count for a particular date. \n',
                                                reply_markup=self.genKeyboard())
                    elif chkNConv(msg['text']) == u'/query' or chkNConv(msg['text']) == u'某日係 day 幾':
                        self._convert_type = ConverType.asking_date
                        self.sender.sendMessage(text=u'邊日呀 ?', reply_markup={'hide_keyboard': True})
                    elif chkNConv(msg['text']) == u'/No' :
                        self._convert_type = ConverType.nothing
                        self.sender.sendMessage(text=u'Bye !',
                                                reply_markup=self.genKeyboard())
                    elif chkNConv(msg['text']) == u'有野要買':
                        self._convert_type = ConverType.adding_to_buy
                        self.sender.sendMessage(text=u'買乜？', reply_markup={'hide_keyboard': True})
                    elif chkNConv(msg['text']) == u'有乜未買？':
                        self.sender.sendMessage(text=self.genBuyList(), reply_markup=self.genKeyboard())
                    else:
                        self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                     u'你試多次啦。或者用 /help 我就教你用。')
                elif self._convert_type == ConverType.asking_date:
                    try :
                        self.sender.sendMessage(text=u'For ' + chkNConv(msg['text']) + u', It is my day ' +
                                                     chkNConv(str(Jinny.getNumOfDaysSpecific(msg['text']))) +
                                                     u'. Any other date to ask ? /No ?')
                    except BaseException :
                        self.sender.sendMessage(text=u'Not a data that I can understand ! \n' +
                                                     u'What date (YYYYMMDD) ? \nOr you done ? ( /No )')
                elif self._convert_type == ConverType.adding_to_buy:
                    if chkNConv(msg['text']) == u'/done':
                        self._convert_type = ConverType.nothing
                        self.doneWithBuyList()
                        self.writeListToFile()
                    else:
                        to_buy_list.append(chkNConv(msg['text']))
                        self.sender.sendMessage(text=u'仲有冇？如果冇，就用 /done 完結。', reply_markup={'hide_keyboard': True})
                else:
                    self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                 u'你試多次啦。或者用 /help 我就教你用。')

            else:
                self.sender.sendMessage(text=u'你邊位呀？')
        else:
            raise telepot.BadFlavor(msg)

        print(u'to buy list: ' + self.genBuyList())

TOKEN = sys.argv[1]  # get token from command-line

with open(name=file_name, mode='r') as f:
    lines = f.read().decode('utf-8').splitlines()
f.close()
to_buy_list = lines

bot = telepot.DelegatorBot(TOKEN, [
    (per_chat_id(), create_open(AngryJinny, timeout=30)),])
print('Listening ...')
bot.notifyOnMessage(run_forever=True)

<|endoftext|>


<commit_before>from ldap import *
from twisted.internet import threads

class Connection(object):

    def __init__(self, uri, *args, **kwargs):
        self._ldap = initialize(uri, *args, **kwargs)

    def add(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.add_ext_s, *args, **kwargs)

    def bind(self, who=None, cred=None, method=None):
        if who is None:
            return threads.deferToThread(self._ldap.simple_bind_s)
        elif who and cred is None:
            return threads.deferToThread(self._ldap.simple_bind_s, who)
        elif who and cred and method is None:
            return threads.deferToThread(self._ldap.simple_bind_s, who, cred)
        else:
            return threads.deferToThread(self._ldap.bind_s, who, cred, method)

    def compare(self, *args, **kargs):
        return threads.deferToThread(self._ldap.compare_ext_s, *args, **kwargs)

    def delete(self, *args, **kargs):
        return threads.deferToThread(self._ldap.delete_ext_s, *args, **kwargs)

    def modify(self, *args, **kargs):
        return threads.deferToThread(self._ldap.modify_ext_s, *args, **kwargs)

    def passwd(self, *args, **kargs):
        return threads.deferToThread(self._ldap.passwd_s, *args, **kwargs)

    def rename(self, *args, **kargs):
        return threads.deferToThread(self._ldap.rename_s, *args, **kwargs)

    def sasl_interactive_bind(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.sasl_interactive_bind_s, *args, **kwargs)

    def start_tls(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.start_tls, *args, **kwargs)

    def search(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.search_s, *args, **kwargs)

    def unbind(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.unbind_ext_s, *args, **kwargs)

    def whoami(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.whoami_s, *args, **kwargs)
<commit_msg>Fix various issues and add debug option<commit_after># Copyright (c) 2010, Silas Sewell
# All rights reserved.
#
# This file is subject to the MIT License (see LICENSE file).

from ldap import *
from twisted.internet import threads

class Connection(object):

    def __init__(self, uri, *args, **kwargs):
        self.debug = kwargs.pop('debug', False)
        self._ldap = initialize(uri, *args, **kwargs)
        if self.debug:
            self._ldap.set_option(ldap.OPT_DEBUG_LEVEL, 255)

    def add(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.add_ext_s, *args, **kwargs)

    def bind(self, who=None, cred=None, method=None):
        if who is None:
            return threads.deferToThread(self._ldap.simple_bind_s)
        elif who and cred is None:
            return threads.deferToThread(self._ldap.simple_bind_s, who)
        elif who and cred and method is None:
            return threads.deferToThread(self._ldap.simple_bind_s, who, cred)
        else:
            return threads.deferToThread(self._ldap.bind_s, who, cred, method)

    def compare(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.compare_ext_s, *args, **kwargs)

    def delete(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.delete_ext_s, *args, **kwargs)

    def modify(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.modify_ext_s, *args, **kwargs)

    def passwd(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.passwd_s, *args, **kwargs)

    def rename(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.rename_s, *args, **kwargs)

    def sasl_interactive_bind(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.sasl_interactive_bind_s, *args, **kwargs)

    def start_tls(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.start_tls_s, *args, **kwargs)

    def search(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.search_s, *args, **kwargs)

    def unbind(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.unbind_ext_s, *args, **kwargs)

    def whoami(self, *args, **kwargs):
        return threads.deferToThread(self._ldap.whoami_s, *args, **kwargs)
<|endoftext|>


<commit_before>import gevent
from gevent import monkey
monkey.patch_all()
import urllib2
from bs4 import BeautifulSoup
# non-fatigue robots :)
import time
from datetime import datetime

import pymongo
c = pymongo.Connection()
db = c["urls"]

c.drop_database("wallpapers")
db2 = c["wallpapers"]

def phostgrab(url):
    """fungsi ini akan mendownload 10 image tiap kali dijalankan."""
    try:
        req = urllib2.Request(url, headers={
        "Referer": "http://pichost.me",
        "User-agent": "Mozilla/5.0",
        })
        html = urllib2.urlopen(req).read()
        soup = BeautifulSoup(html)
        h1 = soup.find("h1").getText().replace(" ", "_").lower()
        div = soup.find("div", attrs={"class": "box1"})

        # this is full-path url to download
        imgurl = div.find("a", href=True)["href"]

        # get fileinfo
        response = urllib2.urlopen(imgurl)
        filetype = response.info()["Content-Type"].split("/")[-1]
        filesize = response.info()["Content-Length"]

        # pastikan dulu bahwa filetype itu image
        # if blabla
        with open("/home/banteng/Desktop/pichost/" + h1 + "_" + imgurl.split("/")[-1].split(".")[0] + "." + filetype, "wb") as f:
            f.write(response.read())

        # insert into mongodb
        db2.wallpaper.insert({
            'title': h1,
            'url': url,
            'filetype': filetype,
            'filesize': filesize,
            })

    except:
        pass

# cari data secara random dari dabatase "urls" yang memiliki status 0
from random import randint

lendata = db.url.find().count()

rndnum = randint(0, lendata-10)

urls = [i['page'] for i in db.url.find({'status': 0}).skip(rndnum).limit(10)]
jobs = [gevent.spawn(phostgrab, url) for url in urls]
gevent.joinall(jobs)

# setelah itu looping urls, ubah status dari 0 menjadi 1
# setelah itu thumbnail
# done!
print "done, now thumbnailing"

for url in urls:
    db.url.update({"page": url}, {"$set": {"status": 1}})

print "status updated to 1"

# sementara kita kembalikan lagi ke 0
for url in urls:
    db.url.update({"status": 1}, {"$set": {"status": 0}})

import os
try:
    from imgtools.thumbnailer import Thumbnailer
    t = Thumbnailer()
except:
    print 'error importing'

from PIL import Image

for image in os.listdir("/home/banteng/Desktop/pichost"):
    try:
        im = Image.open("/home/banteng/Desktop/pichost/" + image)

        if im.size[0] >= 1920 and im.size[0]/float(im.size[1]) >= 1.6:
            print im.size
            # resize and crop
            try:
                t.resize_and_crop("/home/banteng/Desktop/pichost/" + image,
                                      "/home/banteng/Desktop/thumb/thumb_" + image,
                                      (250, 188),
                                      'middle')
            except:
                print 'error thumbnailing'
            # input into mongo
    except:
        print 'gagal'
        continue

print "job all done"
<commit_msg>code edit<commit_after>import gevent
from gevent import monkey
monkey.patch_all()
import urllib2
from bs4 import BeautifulSoup
import time
from datetime import datetime
import os
import pymongo
from PIL import Image

try:
    from imgtools.thumbnailer import Thumbnailer
    t = Thumbnailer()
except:
    print 'error importing'

# database things
c = pymongo.Connection()
db = c["urls"]

c.drop_database("wallpapers")
db2 = c["wallpapers"]

def phostgrab(url):
    """fungsi ini akan mendownload 10 image tiap kali dijalankan."""
    try:
        req = urllib2.Request(url, headers={
        "Referer": "http://pichost.me",
        "User-agent": "Mozilla/5.0",
        })
        html = urllib2.urlopen(req).read()
        soup = BeautifulSoup(html)
        h1 = soup.find("h1").getText().replace(" ", "_").lower()
        div = soup.find("div", attrs={"class": "box1"})

        # this is full-path url to download
        imgurl = div.find("a", href=True)["href"]

        # get fileinfo
        response = urllib2.urlopen(imgurl)
        filetype = response.info()["Content-Type"].split("/")[-1]
        filesize = response.info()["Content-Length"]

        # pastikan dulu bahwa filetype itu image
        # if blabla
        with open("/home/banteng/Desktop/pichost/" + h1 + "_" + imgurl.split("/")[-1].split(".")[0] + "." + filetype, "wb") as f:
            f.write(response.read())

        # insert into mongodb
        db2.wallpaper.insert({
            'title': h1,
            'url': url,
            'filetype': filetype,
            'filesize': filesize,
            })

    except:
        pass

# cari data secara random dari dabatase "urls" yang memiliki status 0
from random import randint
lendata = db.url.find().count()
rndnum = randint(0, lendata-10)

urls = [i['page'] for i in db.url.find({'status': 0}).skip(rndnum).limit(10)]
jobs = [gevent.spawn(phostgrab, url) for url in urls]
gevent.joinall(jobs)

# setelah itu looping urls, ubah status dari 0 menjadi 1
for url in urls:
    db.url.update({"page": url}, {"$set": {"status": 1}})

# sementara kita kembalikan lagi ke 0
for url in urls:
    db.url.update({"status": 1}, {"$set": {"status": 0}})

# setelah itu thumbnail

for image in os.listdir("/home/banteng/Desktop/pichost"):
    try:
        im = Image.open("/home/banteng/Desktop/pichost/" + image)

        if im.size[0] >= 1920 and im.size[0]/float(im.size[1]) >= 1.6:
            # resize and crop
            t.resize_and_crop(
                "/home/banteng/Desktop/pichost/" + image,
                "/home/banteng/Desktop/thumb/thumb_" + image,
                (250, 188),
                'middle')
    except:
        print 'gagal'
        continue

print "job all done"
# done!
<|endoftext|>