<commit_before># -*- coding: utf-8; -*-

from insales.parsing import parse
from insales.composing import compose
from insales.connection import Connection

class InSalesApi(object):

    arrays = {
        'variants'              : 'variant',
        'variants-attributes'   : 'variant',
        'options'               : 'option',
        'option-names'          : 'option-name',
        'option-values'         : 'option-value',
        'orders'                : 'order',
        'order-lines-attributes': 'order-line-attributes',
        'images'                : 'image',
    }

    @classmethod
    def from_credentials(cls, account, api_key, password, **kwargs):
        return cls(Connection(account, api_key, password, **kwargs))

    def __init__(self, connection):
        self.connection = connection

    #========================================================================
    # Заказы
    #========================================================================
    def get_orders(self, per_page=25, page=1, updated_since=None):
        qargs = {
            'per_page': per_page,
            'page': page
        }
        if updated_since:
            qargs['updated_since'] = updated_since
        return self._get('/admin/orders.xml', qargs) or []

    def get_order(self, order_id):
        return self._get('/admin/orders/%s.xml' % order_id)

    def update_order(self, order_id, order_data):
        return self._update('/admin/orders/%s.xml' % order_id, order_data, root='order')

    def delete_order(self, order_id):
        self._delete('/admin/orders/%s.xml' % order_id)

    def create_order(self, order_data):
        return self._add('/admin/orders.xml', order_data, root='order')

    def get_order_delivery_variants(self, order_data):
        return self._post('/admin/orders/delivery_variants.xml',
                          order_data, root='order')

    def get_order_payment_gateways(self, order_data):
        return self._post('/admin/orders/payment_gateways.xml',
                          order_data, root='order')

    #========================================================================
    # Поля заказов
    #========================================================================
    def get_orders_fields(self):
        return self._get('/admin/orders/fields.xml')

    #========================================================================
    # Категории на складе
    #========================================================================
    def get_categories(self):
        return self._get('/admin/categories.xml') or []

    def get_category(self, category_id):
        return self._get('/admin/categories/%s.xml' % category_id)

    def add_category(self, category_data):
        return self._add('/admin/categories.xml', category_data, root='category')

    def update_category(self, category_id, category_data):
        return self._update('/admin/categories/%s.xml' % category_id,
                            category_data, root='category')

    def delete_category(self, category_id):
        self._delete('/admin/categories/%s.xml' % category_id)

    #========================================================================
    # Категории на сайте
    #========================================================================
    def get_collections(self):
        return self._get('/admin/collections.xml') or []

    def get_collection(self, collection_id):
        return self._get('/admin/collections/%s.xml' % collection_id)

    def add_collection(self, collection_data):
        return self._add('/admin/collections.xml', collection_data, root='collection')

    def update_collection(self, collection_id, collection_data):
        return self._update('/admin/collections/%s.xml' % collection_id,
                            collection_data, root='collection')

    def delete_collection(self, collection_id):
        self._delete('/admin/collections/%s.xml' % collection_id)

    #========================================================================
    # Свойства товаров
    #========================================================================
    def get_option_names(self):
        return self._get('/admin/option_names.xml') or []

    def get_option_name(self, option_name_id):
        return self._get('/admin/option_names/%s.xml' % option_name_id)

    def add_option_name(self, option_name):
        return self._add('/admin/option_names.xml', option_name, root='option-name')

    def update_option_name(self, option_name_id, option_name_data):
        return self._update('/admin/option_names/%s.xml' % option_name_id,
                            option_name_data, root='option-name')

    def delete_option_name(self, option_name_id):
        self._delete('/admin/option_names/%s.xml' % option_name_id)

    #========================================================================
    # Значения свойств
    #========================================================================
    def get_option_values(self, option_name_id=None):
        if option_name_id:
            path = '/admin/option_names/%s/option_values.xml' % option_name_id
        else:
            path = '/admin/option_values.xml'
        return self._get(path) or []

    def get_option_value(self, option_name_id, option_value_id):
        return self._get('/admin/option_names/%s/option_values/%s.xml' %
                         (option_name_id, option_value_id))

    def add_option_value(self, option_name_id, option_value_data):
        return self._add('/admin/option_names/%s/option_values.xml' % option_name_id,
                         option_value_data, root='option-value')

    def update_option_value(self, option_name_id, option_value_id, option_value_data):
        return self._update('/admin/option_names/%s/option_values/%s.xml' %
                            (option_name_id, option_value_id),
                            option_value_data, root='option_value')

    def delete_option_value(self, option_name_id, option_value_id):
        self._delete('/admin/option_names/%s/option_values/%s.xml' %
                     (option_name_id, option_value_id))

    #========================================================================
    # Товары
    #========================================================================
    def get_products(self, limit=50, page=1, updated_since=None):
        qargs = {
            'limit': limit,
            'page': page,
        }
        if updated_since:
            qargs['updated_since'] = updated_since
        return self._get('/admin/products.xml', qargs) or []

    def get_product(self, product_id):
        return self._get('/admin/products/%s.xml' % product_id)

    def add_product(self, product_data):
        return self._add('/admin/products.xml', product_data, root='product')

    def update_product(self, product_id, product_data):
        return self._update('/admin/products/%s.xml' % product_id, product_data,
                            root='product')

    def delete_product(self, product_id):
        return self._delete('/admin/products/%s.xml' % product_id)

    #========================================================================
    # Модификации товаров
    #========================================================================
    def get_product_variants(self, product_id):
        return self._get('/admin/products/%s/variants.xml' % product_id) or []

    def get_product_variant(self, product_id, variant_id):
        return self._get('/admin/products/%s/variants/%s.xml' % (product_id, variant_id))

    def add_product_variant(self, product_id, variant_data):
        return self._add('/admin/products/%s/variants.xml' % product_id,
                         variant_data, root='variant')

    def update_product_variant(self, product_id, variant_id, variant_data):
        return self._update('/admin/products/%s/variants/%s.xml' %
                            (product_id, variant_id),
                            variant_data, root='variant')

    def delete_product_variant(self, product_id, variant_id):
        self._delete('/admin/products/%s/variants/%s.xml' % (product_id, variant_id))

    #========================================================================
    # Изображения товара
    #========================================================================
    def get_product_images(self, product_id):
        return self._get('/admin/products/%s/images.xml' % product_id) or []

    def get_product_image(self, product_id, image_id):
        return self._get('/admin/products/%s/images/%s.xml' % (product_id, image_id))

    def add_product_image(self, product_id, image_data):
        return self._add('/admin/products/%s/images.xml' % product_id,
                         image_data, root='image')

    def update_product_image(self, product_id, image_id, image_data):
        return self._update('/admin/products/%s/images/%s.xml' % (product_id, image_id),
                            image_data, root='image')

    def delete_product_image(self, product_id, image_id):
        self._delete('/admin/products/%s/images/%s.xml' % (product_id, image_id))

    #========================================================================
    # Размещение товара
    #========================================================================
    def get_collects(self, product_id=None, collection_id=None, page=1):
        qargs = {}
        if product_id:
            qargs['product_id'] = product_id
        if collection_id:
            qargs['collection_id'] = collection_id

        qargs['page'] = page

        return self._get('/admin/collects.xml', qargs) or []

    def add_collect(self, collect_data):
        return self._add('/admin/collects.xml', collect_data, root='collect')

    def update_collect(self, collect_id, collect_data):
        return self._update('/admin/collects/%s.xml' % collect_id,
                            collect_data, root='collect')

    def delete_collect(self, collect_id):
        return self._delete('/admin/collects/%s.xml' % collect_id)

    #========================================================================
    # Аналогичные товары
    #========================================================================
    def get_similars(self, product_id):
        return self._get('/admin/products/%s/similars.xml' % product_id) or []

    def delete_similar(self, product_id, similar_product_id):
        return self._delete('/admin/products/%s/similars/%s.xml' %
                            (product_id, similar_product_id))

    #========================================================================
    # Сопутствующие товары
    #========================================================================
    def get_supplementaries(self, product_id):
        return self._get('/admin/products/%s/supplementaries.xml' % product_id) or []

    def delete_supplementary(self, product_id, supplementary_product_id):
        return self._delete('/admin/products/%s/supplementaries/%s.xml' %
                            (product_id, supplementary_product_id))

    #========================================================================
    # Веб-хуки
    #========================================================================
    def get_webhooks(self):
        return self._get('/admin/webhooks.xml') or []

    def get_webhook(self, webhook_id):
        return self._get('/admin/webhooks/%s.xml' % webhook_id)

    def add_webhook(self, webhook):
        return self._add('/admin/webhooks.xml', webhook, root='webhook')

    def update_webhook(self, webhook_id, webhook_data):
        return self._update('/admin/webhooks/%s.xml' % webhook_id,
                            webhook_data, root='webhook')

    def delete_webhook(self, webhook_id):
        self._delete('/admin/webhooks/%s.xml' % webhook_id)

    #========================================================================
    # Биллинг
    #========================================================================
    def get_recurring_application_charge(self):
        return self._get('/admin/recurring_application_charge.xml')

    def add_recurring_application_charge(self, recurring_application_charge_data):
        return self._add('/admin/recurring_application_charge.xml',
                         recurring_application_charge_data,
                         root='recurring-application-charge')

    def update_recurring_application_charge(self, recurring_application_charge_data):
        return self._update('/admin/recurring_application_charge.xml',
                            recurring_application_charge_data,
                            root='recurring-application-charge')

    #========================================================================
    def _get(self, endpoint, qargs={}):
        return self._req('get', endpoint, qargs)

    def _add(self, endpoint, data, root):
        xml = compose(data, root=root, arrays=self.arrays)
        return self._req('post', endpoint, xml)

    _post = _add

    def _update(self, endpoint, data, root):
        xml = compose(data, root=root, arrays=self.arrays)
        return self._req('put', endpoint, xml)

    def _delete(self, endpoint):
        return self._req('delete', endpoint)

    def _req(self, method, *args, **kwargs):
        response = getattr(self.connection, method)(*args, **kwargs)
        return parse(response)
<commit_msg>fix: add real array symbols<commit_after># -*- coding: utf-8; -*-

from insales.parsing import parse
from insales.composing import compose
from insales.connection import Connection

class InSalesApi(object):

    arrays = {
        'all-tags': 'all-tag',
        'application-actions': 'application-action',
        'application-charges': 'application-charge',
        'application-widgets': 'application-widget',
        'articles': 'article',
        'blogs': 'blog',
        'bonus-system-transactions': 'bonus-system-transaction',
        'categories': 'category',
        'characteristics': 'characteristic',
        'client-groups': 'client-group',
        'clients': 'client',
        'collection-field-values': 'collection-field-value',
        'collection-fields': 'collection-field',
        'collection-filters': 'collection-filter',
        'collection-ids': 'collection-id',
        'collections': 'collection',
        'collections-ids': 'collections-id',
        'collects': 'collect',
        'custom-statuses': 'custom-status',
        'delivery-locations': 'delivery-location',
        'delivery-locations-attributes': 'delivery-locations-attribute',
        'delivery-zones': 'delivery-zone',
        'delivery-zones-attributes': 'delivery-zones-attribute',
        'description-translations': 'description-translation',
        'discount-codes': 'discount-code',
        'discount-collections': 'discount-collection',
        'discount-order-lines-ids': 'discount-order-lines-id',
        'discount-products-ids': 'discount-products-id',
        'discounts': 'discount',
        'discounts-attributes': 'discounts-attribute',
        'domains': 'domain',
        'errors': 'error',
        'field-options': 'field-option',
        'field-options-attributes': 'field-options-attribute',
        'field-values-attributes': 'field-values-attribute',
        'fields': 'field',
        'fields-values': 'fields-value',
        'fields-values-attributes': 'fields-values-attribute',
        'files': 'file',
        'images': 'image',
        'js-tags': 'js-tag',
        'locations': 'location',
        'locations-attributes': 'locations-attribute',
        'menu-items': 'menu-item',
        'menus': 'menu',
        'nil-classes': 'nil-class',
        'option-names': 'option-name',
        'option-values': 'option-value',
        'options': 'option',
        'options-attributes': 'options-attribute',
        'order-changes': 'order-change',
        'order-lines': 'order-line',
        'order-lines-attributes': 'order-lines-attribute',
        'orders': 'order',
        'outlets': 'outlet',
        'pages': 'page',
        'payment-delivery-variants': 'payment-delivery-variant',
        'payment-delivery-variants-attributes': 'payment-delivery-variants-attribute',
        'pick-up-sources': 'pick-up-source',
        'price-kinds': 'price-kind',
        'prices': 'price',
        'prices-in-site-currency': 'price-in-site-currency',
        'product-bundle-components': 'product-bundle-component',
        'product-bundle-components-attributes': 'product-bundle-components-attribute',
        'product-field-values': 'product-field-value',
        'product-field-values-attributes': 'product-field-values-attribute',
        'products': 'product',
        'properties': 'property',
        'properties-attributes': 'properties-attribute',
        'redirects': 'redirect',
        'related-products': 'related-product',
        'reviews': 'review',
        'rules': 'rule',
        'rules-attributes': 'rules-attribute',
        'similar-ids': 'similar-id',
        'stock-currencies': 'stock-currency',
        'supplementary-ids': 'supplementary-id',
        'tags': 'tag',
        'tariffs': 'tariff',
        'tariffs-attributes': 'tariffs-attribute',
        'title-translations': 'title-translation',
        'variant-field-values': 'variant-field-value',
        'variant-field-values-attributes': 'variant-field-values-attribute',
        'variants': 'variant',
        'variants-attributes': 'variants-attribute',
        'warnings': 'warning',
        'webhooks': 'webhook'
    }

    @classmethod
    def from_credentials(cls, account, api_key, password, **kwargs):
        return cls(Connection(account, api_key, password, **kwargs))

    def __init__(self, connection):
        self.connection = connection

    #========================================================================
    # Заказы
    #========================================================================
    def get_orders(self, per_page=25, page=1, updated_since=None):
        qargs = {
            'per_page': per_page,
            'page': page
        }
        if updated_since:
            qargs['updated_since'] = updated_since
        return self._get('/admin/orders.xml', qargs) or []

    def get_order(self, order_id):
        return self._get('/admin/orders/%s.xml' % order_id)

    def update_order(self, order_id, order_data):
        return self._update('/admin/orders/%s.xml' % order_id, order_data, root='order')

    def delete_order(self, order_id):
        self._delete('/admin/orders/%s.xml' % order_id)

    def create_order(self, order_data):
        return self._add('/admin/orders.xml', order_data, root='order')

    def get_order_delivery_variants(self, order_data):
        return self._post('/admin/orders/delivery_variants.xml',
                          order_data, root='order')

    def get_order_payment_gateways(self, order_data):
        return self._post('/admin/orders/payment_gateways.xml',
                          order_data, root='order')

    #========================================================================
    # Поля заказов
    #========================================================================
    def get_orders_fields(self):
        return self._get('/admin/orders/fields.xml')

    #========================================================================
    # Категории на складе
    #========================================================================
    def get_categories(self):
        return self._get('/admin/categories.xml') or []

    def get_category(self, category_id):
        return self._get('/admin/categories/%s.xml' % category_id)

    def add_category(self, category_data):
        return self._add('/admin/categories.xml', category_data, root='category')

    def update_category(self, category_id, category_data):
        return self._update('/admin/categories/%s.xml' % category_id,
                            category_data, root='category')

    def delete_category(self, category_id):
        self._delete('/admin/categories/%s.xml' % category_id)

    #========================================================================
    # Категории на сайте
    #========================================================================
    def get_collections(self):
        return self._get('/admin/collections.xml') or []

    def get_collection(self, collection_id):
        return self._get('/admin/collections/%s.xml' % collection_id)

    def add_collection(self, collection_data):
        return self._add('/admin/collections.xml', collection_data, root='collection')

    def update_collection(self, collection_id, collection_data):
        return self._update('/admin/collections/%s.xml' % collection_id,
                            collection_data, root='collection')

    def delete_collection(self, collection_id):
        self._delete('/admin/collections/%s.xml' % collection_id)

    #========================================================================
    # Свойства товаров
    #========================================================================
    def get_option_names(self):
        return self._get('/admin/option_names.xml') or []

    def get_option_name(self, option_name_id):
        return self._get('/admin/option_names/%s.xml' % option_name_id)

    def add_option_name(self, option_name):
        return self._add('/admin/option_names.xml', option_name, root='option-name')

    def update_option_name(self, option_name_id, option_name_data):
        return self._update('/admin/option_names/%s.xml' % option_name_id,
                            option_name_data, root='option-name')

    def delete_option_name(self, option_name_id):
        self._delete('/admin/option_names/%s.xml' % option_name_id)

    #========================================================================
    # Значения свойств
    #========================================================================
    def get_option_values(self, option_name_id=None):
        if option_name_id:
            path = '/admin/option_names/%s/option_values.xml' % option_name_id
        else:
            path = '/admin/option_values.xml'
        return self._get(path) or []

    def get_option_value(self, option_name_id, option_value_id):
        return self._get('/admin/option_names/%s/option_values/%s.xml' %
                         (option_name_id, option_value_id))

    def add_option_value(self, option_name_id, option_value_data):
        return self._add('/admin/option_names/%s/option_values.xml' % option_name_id,
                         option_value_data, root='option-value')

    def update_option_value(self, option_name_id, option_value_id, option_value_data):
        return self._update('/admin/option_names/%s/option_values/%s.xml' %
                            (option_name_id, option_value_id),
                            option_value_data, root='option_value')

    def delete_option_value(self, option_name_id, option_value_id):
        self._delete('/admin/option_names/%s/option_values/%s.xml' %
                     (option_name_id, option_value_id))

    #========================================================================
    # Товары
    #========================================================================
    def get_products(self, limit=50, page=1, updated_since=None):
        qargs = {
            'limit': limit,
            'page': page,
        }
        if updated_since:
            qargs['updated_since'] = updated_since
        return self._get('/admin/products.xml', qargs) or []

    def get_product(self, product_id):
        return self._get('/admin/products/%s.xml' % product_id)

    def add_product(self, product_data):
        return self._add('/admin/products.xml', product_data, root='product')

    def update_product(self, product_id, product_data):
        return self._update('/admin/products/%s.xml' % product_id, product_data,
                            root='product')

    def delete_product(self, product_id):
        return self._delete('/admin/products/%s.xml' % product_id)

    #========================================================================
    # Модификации товаров
    #========================================================================
    def get_product_variants(self, product_id):
        return self._get('/admin/products/%s/variants.xml' % product_id) or []

    def get_product_variant(self, product_id, variant_id):
        return self._get('/admin/products/%s/variants/%s.xml' % (product_id, variant_id))

    def add_product_variant(self, product_id, variant_data):
        return self._add('/admin/products/%s/variants.xml' % product_id,
                         variant_data, root='variant')

    def update_product_variant(self, product_id, variant_id, variant_data):
        return self._update('/admin/products/%s/variants/%s.xml' %
                            (product_id, variant_id),
                            variant_data, root='variant')

    def delete_product_variant(self, product_id, variant_id):
        self._delete('/admin/products/%s/variants/%s.xml' % (product_id, variant_id))

    #========================================================================
    # Изображения товара
    #========================================================================
    def get_product_images(self, product_id):
        return self._get('/admin/products/%s/images.xml' % product_id) or []

    def get_product_image(self, product_id, image_id):
        return self._get('/admin/products/%s/images/%s.xml' % (product_id, image_id))

    def add_product_image(self, product_id, image_data):
        return self._add('/admin/products/%s/images.xml' % product_id,
                         image_data, root='image')

    def update_product_image(self, product_id, image_id, image_data):
        return self._update('/admin/products/%s/images/%s.xml' % (product_id, image_id),
                            image_data, root='image')

    def delete_product_image(self, product_id, image_id):
        self._delete('/admin/products/%s/images/%s.xml' % (product_id, image_id))

    #========================================================================
    # Размещение товара
    #========================================================================
    def get_collects(self, product_id=None, collection_id=None, page=1):
        qargs = {}
        if product_id:
            qargs['product_id'] = product_id
        if collection_id:
            qargs['collection_id'] = collection_id

        qargs['page'] = page

        return self._get('/admin/collects.xml', qargs) or []

    def add_collect(self, collect_data):
        return self._add('/admin/collects.xml', collect_data, root='collect')

    def update_collect(self, collect_id, collect_data):
        return self._update('/admin/collects/%s.xml' % collect_id,
                            collect_data, root='collect')

    def delete_collect(self, collect_id):
        return self._delete('/admin/collects/%s.xml' % collect_id)

    #========================================================================
    # Аналогичные товары
    #========================================================================
    def get_similars(self, product_id):
        return self._get('/admin/products/%s/similars.xml' % product_id) or []

    def delete_similar(self, product_id, similar_product_id):
        return self._delete('/admin/products/%s/similars/%s.xml' %
                            (product_id, similar_product_id))

    #========================================================================
    # Сопутствующие товары
    #========================================================================
    def get_supplementaries(self, product_id):
        return self._get('/admin/products/%s/supplementaries.xml' % product_id) or []

    def delete_supplementary(self, product_id, supplementary_product_id):
        return self._delete('/admin/products/%s/supplementaries/%s.xml' %
                            (product_id, supplementary_product_id))

    #========================================================================
    # Веб-хуки
    #========================================================================
    def get_webhooks(self):
        return self._get('/admin/webhooks.xml') or []

    def get_webhook(self, webhook_id):
        return self._get('/admin/webhooks/%s.xml' % webhook_id)

    def add_webhook(self, webhook):
        return self._add('/admin/webhooks.xml', webhook, root='webhook')

    def update_webhook(self, webhook_id, webhook_data):
        return self._update('/admin/webhooks/%s.xml' % webhook_id,
                            webhook_data, root='webhook')

    def delete_webhook(self, webhook_id):
        self._delete('/admin/webhooks/%s.xml' % webhook_id)

    #========================================================================
    # Биллинг
    #========================================================================
    def get_recurring_application_charge(self):
        return self._get('/admin/recurring_application_charge.xml')

    def add_recurring_application_charge(self, recurring_application_charge_data):
        return self._add('/admin/recurring_application_charge.xml',
                         recurring_application_charge_data,
                         root='recurring-application-charge')

    def update_recurring_application_charge(self, recurring_application_charge_data):
        return self._update('/admin/recurring_application_charge.xml',
                            recurring_application_charge_data,
                            root='recurring-application-charge')

    #========================================================================
    def _get(self, endpoint, qargs={}):
        return self._req('get', endpoint, qargs)

    def _add(self, endpoint, data, root):
        xml = compose(data, root=root, arrays=self.arrays)
        return self._req('post', endpoint, xml)

    _post = _add

    def _update(self, endpoint, data, root):
        xml = compose(data, root=root, arrays=self.arrays)
        return self._req('put', endpoint, xml)

    def _delete(self, endpoint):
        return self._req('delete', endpoint)

    def _req(self, method, *args, **kwargs):
        response = getattr(self.connection, method)(*args, **kwargs)
        return parse(response)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Mission Pinball Framework documentation build configuration
# file, created by sphinx-quickstart on Fri Mar 25 18:39:44 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

try:
    import sphinx_bootstrap_theme
    bootstrap = True

except ImportError:
    bootstrap = False

bootstrap = False
import sphinx_rtd_theme

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    # 'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig',
]

# Add any paths that contain templates here, relative to this directory.

if bootstrap:

    templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'Mission Pinball Framework'
copyright = '2013-2016, The Mission Pinball Framework Team'
author = 'The Mission Pinball Framework Team'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.31'
# The full version, including alpha/beta/rc tags.
release = '0.31'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build', '_not_updated_yet']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'none'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

if bootstrap:
    html_theme = 'bootstrap'
    html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()

    # Theme options are theme-specific and customize the look and feel of a theme
    # further.  For a list of options available for each theme, see the
    # documentation.
    html_theme_options = {

        'navbar_title': "MPF Documentation",
        'navbar_site_name': "All Docs",
        'navbar_pagenav_name': "This Page",
        'globaltoc_depth': -1,
        'bootswatch_theme': "cosmo",
        'navbar_sidebarrel': False,
        'navbar_pagenav': False,
        'source_link_position': False,
        'navbar_fixed_top': False,
    }

    html_add_permalinks = ''

else:
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = '_static/images/mpf-logo-tiny.png'

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {'**': ['localtoc.html']}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'MissionPinballFrameworkdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
'figure_align':'H',

# Examples of font packages: www.icl.utk.edu/~mgates3/docs/latex-fonts.pdf
# None of them look good, so we'll just comment out to use the default
# 'fontpkg':'\\usepackage{bookman}',

}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'MissionPinballFramework.tex', 'Mission Pinball Framework Documentation',
     'The Mission Pinball Framework Team', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
latex_show_pagerefs = True

# If true, show URL addresses after external links.
latex_show_urls = 'inline'

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'missionpinballframework', 'Mission Pinball Framework Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'MissionPinballFramework', 'Mission Pinball Framework Documentation',
     author, 'MissionPinballFramework', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
#epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
#epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}
<commit_msg>update version to 0.32<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Mission Pinball Framework documentation build configuration
# file, created by sphinx-quickstart on Fri Mar 25 18:39:44 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

try:
    import sphinx_bootstrap_theme
    bootstrap = True

except ImportError:
    bootstrap = False

bootstrap = False
import sphinx_rtd_theme

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    # 'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig',
]

# Add any paths that contain templates here, relative to this directory.

if bootstrap:

    templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'Mission Pinball Framework'
copyright = '2013-2016, The Mission Pinball Framework Team'
author = 'The Mission Pinball Framework Team'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.32'
# The full version, including alpha/beta/rc tags.
release = '0.32'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build', '_not_updated_yet']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'none'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.

if bootstrap:
    html_theme = 'bootstrap'
    html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()

    # Theme options are theme-specific and customize the look and feel of a theme
    # further.  For a list of options available for each theme, see the
    # documentation.
    html_theme_options = {

        'navbar_title': "MPF Documentation",
        'navbar_site_name': "All Docs",
        'navbar_pagenav_name': "This Page",
        'globaltoc_depth': -1,
        'bootswatch_theme': "cosmo",
        'navbar_sidebarrel': False,
        'navbar_pagenav': False,
        'source_link_position': False,
        'navbar_fixed_top': False,
    }

    html_add_permalinks = ''

else:
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = '_static/images/mpf-logo-tiny.png'

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {'**': ['localtoc.html']}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'MissionPinballFrameworkdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
'figure_align':'H',

# Examples of font packages: www.icl.utk.edu/~mgates3/docs/latex-fonts.pdf
# None of them look good, so we'll just comment out to use the default
# 'fontpkg':'\\usepackage{bookman}',

}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'MissionPinballFramework.tex', 'Mission Pinball Framework Documentation',
     'The Mission Pinball Framework Team', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
latex_show_pagerefs = True

# If true, show URL addresses after external links.
latex_show_urls = 'inline'

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'missionpinballframework', 'Mission Pinball Framework Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'MissionPinballFramework', 'Mission Pinball Framework Documentation',
     author, 'MissionPinballFramework', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
#epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
#epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
insta_raider.py

usage: insta_raider.py [-h] -u USERNAME

@amirkurtovic

"""
import argparse
import logging
import os
import os.path as op
import re
import email.utils as eut
import requests
import calendar
try:
    from requests.packages.urllib3.exceptions import InsecurePlatformWarning
except ImportError:
    from urllib3.exceptions import InsecurePlatformWarning
import time
import warnings
import selenium.webdriver as webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import NoSuchElementException
import json
from datetime import datetime
try:
    from gi.repository import GExiv2
except ImportError:
    GExiv2 = None

warnings.filterwarnings("ignore", category=InsecurePlatformWarning)

class PrivateUserError(Exception):
    """Raised if the profile is found to be private"""

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

class InstaRaider(object):

    def __init__(self, username, directory, num_to_download=None,
                 log_level='info', use_metadata=False, get_videos=False):
        self.username = username
        self.profile_url = self.get_url(username)
        self.directory = directory
        self.PAUSE = 1
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        self.headers = {'User-Agent': self.user_agent}
        self.html_source = None
        self.log_level = getattr(logging, log_level.upper())
        self.setup_logging(self.log_level)
        self.use_metadata = use_metadata
        self.get_videos = get_videos
        self.set_num_posts(num_to_download)
        self.setup_webdriver()

    def __del__(self):
        if self.webdriver:
            self.webdriver.close()

    def get_url(self, path):
        return urlparse.urljoin('https://instagram.com', path)

    def set_num_posts(self, num_to_download=None):
        self.num_posts = int(self.get_posts_count(self.profile_url) or 0)
        self.num_to_download = num_to_download

    def setup_logging(self, level=logging.INFO):
        self.logger = logging.getLogger('instaraider')
        self.logger.addHandler(logging.StreamHandler())
        self.logger.setLevel(logging.INFO)

    def log(self, *strings, **kwargs):
        level = kwargs.pop('level', logging.INFO)
        self.logger.log(level, u' '.join(str(s) for s in strings))

    def setup_webdriver(self):
        self.profile = webdriver.FirefoxProfile()
        self.profile.set_preference("general.useragent.override", self.user_agent)
        self.webdriver = webdriver.Firefox(self.profile)
        self.webdriver.set_window_size(480, 320)
        self.webdriver.set_window_position(800, 0)

    def get_posts_count(self, url):
        """
        Given a url to Instagram profile, return number of photos posted
        """
        response = requests.get(url)
        counts_code = re.search(r'\"media":{"count":\d+', response.text)
        if not counts_code:
            return None
        return re.findall(r'\d+', counts_code.group())[0]

    def log_in_user(self):
        driver = self.webdriver
        self.log('You need to login to access this profile.',
                 'Redirecting you to the login page in the browser.',
                 level=logging.WARN)
        driver.get(self.get_url('accounts/login/'))

        # Wait until user has been successfully logged in and redirceted
        # to his/her feed.
        WebDriverWait(driver, 60).until(
            expected_conditions.presence_of_element_located(
                (By.CSS_SELECTOR, '.-cx-PRIVATE-FeedPage__feed'),
            )
        )

        self.log('User successfully logged in.', level=logging.INFO)
        self.set_num_posts()  # Have to set this again
        driver.get(self.profile_url)

    def load_instagram(self):
        """
        Using Selenium WebDriver, load Instagram page to get page source

        """
        self.log(self.username, 'has', self.num_posts, 'posts on Instagram.')
        if self.num_to_download is not None:
            self.log("The first", self.num_to_download, "of them will be downloaded.")

        num_to_download = self.num_to_download or self.num_posts
        driver = self.webdriver
        # load Instagram profile and wait for PAUSE
        self.log("Loading Instagram profile...")
        driver.get(self.profile_url)
        driver.implicitly_wait(self.PAUSE)

        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        try:
            el = driver.find_element_by_css_selector(
                '.-cx-PRIVATE-ProfilePage__advisoryMessageHeader'
            )
        except NoSuchElementException:
            pass
        else:
            self.log_in_user()

        if (num_to_download > 24):
            scroll_to_bottom = self.get_scroll_count(num_to_download)
            element = driver.find_element_by_css_selector('div.-cx-PRIVATE-AutoloadingPostsGrid__moreLoadingIndicator a')
            driver.implicitly_wait(self.PAUSE)
            element.click()

            for y in range(int(scroll_to_bottom)):
                self.scroll_page(driver)

        # After load all profile photos, retur, source to download_photos()
        time.sleep(1)
        source = driver.page_source

        return source

    def scroll_page(self, driver):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(0.2)
        driver.execute_script("window.scrollTo(0, 0);")

    def get_scroll_count(self, count):
        return (int(count) - 24) / 12 + 1

    def validate(self):
        """
        returns True if Instagram username is valid
        """
        req = requests.get(self.profile_url)

        try:
            req.raise_for_status()
        except:
            self.log('User', self.username, 'is not valid.',
                     level=logging.ERROR)
            return False

        if not self.num_posts:
            self.log('User', self.username, 'has no photos to download.',
                     level=logging.ERROR)
            return False
        return True

    def save_photo(self, photo_url, photo_name):
        image_request = requests.get(photo_url, headers=self.headers)
        image_data = image_request.content
        with open(photo_name, 'wb') as fp:
            fp.write(image_data)
        return image_request.headers

    def download_photos(self):
        """
        Given source code for loaded Instagram page,
        extract all hrefs and download full-resolution photos

        source: HTML source code of Instagram profile papge
        """
        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()

        # check if directory exists, if not, make it
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        # index for progress bar
        photos_saved = 0
        self.log("Saving photos to", self.directory)

        links = self.find_links()

        for link in links:
            photo_url = link['display_src']
            photo_url = photo_url.replace('\\', '')
            photo_url = re.sub(r'/s\d+x\d+/', '/', photo_url)
            caption = link.get('caption')
            date_time = link.get('date_time')
            photo_basename = op.basename(photo_url)
            photo_name = op.join(self.directory, photo_basename)

            # save full-resolution photo if its new
            if not op.isfile(photo_name):
                headers = self.save_photo(photo_url, photo_name)
                photos_saved += 1
                self.log('Downloaded file {}/{} ({}).'.format(
                    photos_saved, num_to_download, photo_basename))
                # put info from Instagram post into image metadata
                if self.use_metadata:
                    self.add_metadata(photo_name, caption, date_time)
                if "last-modified" in headers:
                    modtime = calendar.timegm(eut.parsedate(headers["last-modified"]))
                    os.utime(photo_name, (modtime, modtime))
            else:
                self.log('Skipping file', photo_name, 'as it already exists.')

            if photos_saved >= num_to_download:
                break

        self.log('Saved', photos_saved, 'files to', self.directory)

    def find_links(self):
        """
        Find all image urls/metadata in html_source.

        Returns a list of dicts.
        e.g., [{'display_src': 'http://image.url',
                'caption': 'some text from Instagram post',
                'date_time': '1448420058.0'},
               {...},]
        'display_src' must be present in each dict;
            the other keys are optional.
        """
        photos = []
        if self.use_metadata:
            if not GExiv2:
                self.use_metadata = False
                self.log('GExiv2 python module not found.',
                         'Images will not be tagged.')
        try:
            json_data = re.search(r'(?s)<script [^>]*>window\._sharedData'
                                  r'.*?"nodes".+?</script>',
                                  self.html_source)
            json_data = re.search(r'{.+}', json_data.group(0))
            json_data = json.loads(json_data.group(0))
            photos = list(gen_dict_extract('nodes', json_data))[0]
            # find profile_pic
            profile_pic = list(gen_dict_extract('profile_pic_url', json_data))
            if profile_pic:
                # todo (possible):
                #   add a key in the dict to indicate this is profile_pic.
                #   then we could also name the file "profile.jpg" or similar
                #   and also not include it in photos_saved so if user
                #   uses -n N to download some number of images, he still gets
                #   the first N images rather than N-1 images plus
                #   profile_pic
                profile_pic = [{'display_src': p} for p in profile_pic[:1]]
                photos = profile_pic + photos
        except:
            if self.use_metadata:
                self.use_metadata = False
                self.log('Could not find any image metadata.',
                         'Photos will not be tagged.')
            links = re.finditer(r'src="([https]+:...[\/\w \.-]*..[\/\w \.-]*'
                                r'..[\/\w \.-]*..[\/\w \.-].jpg)',
                                self.html_source)
            photos = [{'display_src': m.group(1)} for m in links]
        for photo in photos:
            date = photo.get('date')
            if date:
                try:
                    photo['date_time'] = datetime.fromtimestamp(date)
                except:
                    photo['date_time'] = None
        return photos

    def download_videos(self):
        """
        Given source code for loaded Instagram page:
        - discover all video wrapper links
        - activate all links to load video url
        - extract and download video url
        """

        if not self.get_videos:
            return;

        # We need to use the driver to query the video wrappers
        driver = self.webdriver

        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        videos_saved = 0
        self.log("Saving videos to", self.directory)

        # Find all of the video wrappers
        video_wrapper_elements = driver.find_elements_by_xpath('.//*[@id="react-root"]/section/main/article/div/div[1]/div/a[.//*[@Class="-cx-PRIVATE-PostsGridItem__videoIndicatorWrapper"]]')
        video_wrapper_urls = [link.get_attribute('href') for link in video_wrapper_elements]

        for video_wrapper in video_wrapper_urls:
            # Fetch the link of the video wrapper
            driver.get(video_wrapper)

            # Wait until the real video appears
            WebDriverWait(driver, 60).until(
                expected_conditions.presence_of_all_elements_located(
                    (By.CLASS_NAME, '-cx-PRIVATE-DeclarativeVideo__video')
                )
            )

            # Get the real video, since only 1 video can be clicked on at a time, we only expect there to be a single result
            video_elements = driver.find_elements_by_class_name('-cx-PRIVATE-DeclarativeVideo__video')
            if len(video_elements) > 0:
                video_url = video_elements[0].get_attribute('src')
                video_name = op.join(self.directory, video_url.split('/')[len(video_url.split('/')) - 1])

                if not op.isfile(video_name):
                    self.save_photo(video_url, video_name)
                    videos_saved += 1
                    self.log('Downloaded file {}/{} ({}).'.format(
                        videos_saved, num_to_download, op.basename(video_name)))
                else:
                    self.log('Skipping file', video_name, 'as it already exists.')

                if videos_saved >= num_to_download:
                    break

        self.log('Saved', videos_saved, 'videos to', self.directory)

    def add_metadata(self, photo_name, caption, date_time):
        """
        Tag downloaded photos with metadata from associated Instagram post.

        If GExiv2 is not installed, do nothing.
        """
        if GExiv2:
            if caption or date_time:
                # todo: improve error handling
                try:
                    exif = GExiv2.Metadata(photo_name)
                    if caption:
                        try:
                            exif.set_comment(caption)
                        except:
                            self.log('Error setting image caption metadata.')
                    if date_time:
                        try:
                            exif.set_date_time(date_time)
                        except:
                            self.log('Error setting image date metadata.')
                    exif.save_file()
                except:
                    pass

def gen_dict_extract(key, var):
    """
    Recursively search for given dict key in nested dicts.

    from http://stackoverflow.com/a/29652561
    author: hexerei software
    """
    if hasattr(var,'iteritems'):
        for k, v in var.iteritems():
            if k == key:
                yield v
            if isinstance(v, dict):
                for result in gen_dict_extract(key, v):
                    yield result
            elif isinstance(v, list):
                for d in v:
                    for result in gen_dict_extract(key, d):
                        yield result

def main():
    # parse arguments
    parser = argparse.ArgumentParser(description='InstaRaider')
    parser.add_argument('username', help='Instagram username')
    parser.add_argument('directory', help='Where to save the images')
    parser.add_argument('-n', '--num-to-download',
                        help='Number of posts to download', type=int)
    parser.add_argument('-l', '--log-level', help="Log level", default='info')
    parser.add_argument('-m', '--add_metadata',
                        help=("Add metadata (caption/date) from Instagram "
                              "post into downloaded images' exif tags "
                              "(requires GExiv2 python module)"),
                        action='store_true', dest='use_metadata')
    parser.add_argument('-v', '--get_videos',
                        help=("Download videos"),
                        action='store_true', dest='get_videos')
    args = parser.parse_args()
    username = args.username
    directory = op.expanduser(args.directory)

    raider = InstaRaider(username, directory,
                         num_to_download=args.num_to_download,
                         log_level=args.log_level,
                         use_metadata=args.use_metadata,
                         get_videos=args.get_videos)

    if not raider.validate():
        return

    raider.download_photos()
    raider.download_videos()

if __name__ == '__main__':
    main()
<commit_msg>Fix video and load more classes<commit_after>#!/usr/bin/env python
"""
insta_raider.py

usage: insta_raider.py [-h] -u USERNAME

@amirkurtovic

"""
import argparse
import logging
import os
import os.path as op
import re
import email.utils as eut
import requests
import calendar
try:
    from requests.packages.urllib3.exceptions import InsecurePlatformWarning
except ImportError:
    from urllib3.exceptions import InsecurePlatformWarning
import time
import warnings
import selenium.webdriver as webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import NoSuchElementException
import json
from datetime import datetime
try:
    from gi.repository import GExiv2
except ImportError:
    GExiv2 = None

warnings.filterwarnings("ignore", category=InsecurePlatformWarning)

class PrivateUserError(Exception):
    """Raised if the profile is found to be private"""

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

class InstaRaider(object):

    def __init__(self, username, directory, num_to_download=None,
                 log_level='info', use_metadata=False, get_videos=False):
        self.username = username
        self.profile_url = self.get_url(username)
        self.directory = directory
        self.PAUSE = 1
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        self.headers = {'User-Agent': self.user_agent}
        self.html_source = None
        self.log_level = getattr(logging, log_level.upper())
        self.setup_logging(self.log_level)
        self.use_metadata = use_metadata
        self.get_videos = get_videos
        self.set_num_posts(num_to_download)
        self.setup_webdriver()

    def __del__(self):
        if self.webdriver:
            self.webdriver.close()

    def get_url(self, path):
        return urlparse.urljoin('https://instagram.com', path)

    def set_num_posts(self, num_to_download=None):
        self.num_posts = int(self.get_posts_count(self.profile_url) or 0)
        self.num_to_download = num_to_download

    def setup_logging(self, level=logging.INFO):
        self.logger = logging.getLogger('instaraider')
        self.logger.addHandler(logging.StreamHandler())
        self.logger.setLevel(logging.INFO)

    def log(self, *strings, **kwargs):
        level = kwargs.pop('level', logging.INFO)
        self.logger.log(level, u' '.join(str(s) for s in strings))

    def setup_webdriver(self):
        self.profile = webdriver.FirefoxProfile()
        self.profile.set_preference("general.useragent.override", self.user_agent)
        self.webdriver = webdriver.Firefox(self.profile)
        self.webdriver.set_window_size(480, 320)
        self.webdriver.set_window_position(800, 0)

    def get_posts_count(self, url):
        """
        Given a url to Instagram profile, return number of photos posted
        """
        response = requests.get(url)
        counts_code = re.search(r'\"media":{"count":\d+', response.text)
        if not counts_code:
            return None
        return re.findall(r'\d+', counts_code.group())[0]

    def log_in_user(self):
        driver = self.webdriver
        self.log('You need to login to access this profile.',
                 'Redirecting you to the login page in the browser.',
                 level=logging.WARN)
        driver.get(self.get_url('accounts/login/'))

        # Wait until user has been successfully logged in and redirceted
        # to his/her feed.
        WebDriverWait(driver, 60).until(
            expected_conditions.presence_of_element_located(
                (By.CSS_SELECTOR, '.-cx-PRIVATE-FeedPage__feed'),
            )
        )

        self.log('User successfully logged in.', level=logging.INFO)
        self.set_num_posts()  # Have to set this again
        driver.get(self.profile_url)

    def load_instagram(self):
        """
        Using Selenium WebDriver, load Instagram page to get page source

        """
        self.log(self.username, 'has', self.num_posts, 'posts on Instagram.')
        if self.num_to_download is not None:
            self.log("The first", self.num_to_download, "of them will be downloaded.")

        num_to_download = self.num_to_download or self.num_posts
        driver = self.webdriver
        # load Instagram profile and wait for PAUSE
        self.log("Loading Instagram profile...")
        driver.get(self.profile_url)
        driver.implicitly_wait(self.PAUSE)

        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        try:
            el = driver.find_element_by_css_selector(
                '.-cx-PRIVATE-ProfilePage__advisoryMessageHeader'
            )
        except NoSuchElementException:
            pass
        else:
            self.log_in_user()

        if (num_to_download > 24):
            scroll_to_bottom = self.get_scroll_count(num_to_download)
            element = driver.find_element_by_css_selector('div.o98')
            driver.implicitly_wait(self.PAUSE)
            element.click()

            for y in range(int(scroll_to_bottom)):
                self.scroll_page(driver)

        # After load all profile photos, retur, source to download_photos()
        time.sleep(1)
        source = driver.page_source

        return source

    def scroll_page(self, driver):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(0.2)
        driver.execute_script("window.scrollTo(0, 0);")

    def get_scroll_count(self, count):
        return (int(count) - 24) / 12 + 1

    def validate(self):
        """
        returns True if Instagram username is valid
        """
        req = requests.get(self.profile_url)

        try:
            req.raise_for_status()
        except:
            self.log('User', self.username, 'is not valid.',
                     level=logging.ERROR)
            return False

        if not self.num_posts:
            self.log('User', self.username, 'has no photos to download.',
                     level=logging.ERROR)
            return False
        return True

    def save_photo(self, photo_url, photo_name):
        image_request = requests.get(photo_url, headers=self.headers)
        image_data = image_request.content
        with open(photo_name, 'wb') as fp:
            fp.write(image_data)
        return image_request.headers

    def download_photos(self):
        """
        Given source code for loaded Instagram page,
        extract all hrefs and download full-resolution photos

        source: HTML source code of Instagram profile papge
        """
        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()

        # check if directory exists, if not, make it
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        # index for progress bar
        photos_saved = 0
        self.log("Saving photos to", self.directory)

        links = self.find_links()

        for link in links:
            photo_url = link['display_src']
            photo_url = photo_url.replace('\\', '')
            photo_url = re.sub(r'/s\d+x\d+/', '/', photo_url)
            caption = link.get('caption')
            date_time = link.get('date_time')
            photo_basename = op.basename(photo_url)
            photo_name = op.join(self.directory, photo_basename)

            # save full-resolution photo if its new
            if not op.isfile(photo_name):
                headers = self.save_photo(photo_url, photo_name)
                photos_saved += 1
                self.log('Downloaded file {}/{} ({}).'.format(
                    photos_saved, num_to_download, photo_basename))
                # put info from Instagram post into image metadata
                if self.use_metadata:
                    self.add_metadata(photo_name, caption, date_time)
                if "last-modified" in headers:
                    modtime = calendar.timegm(eut.parsedate(headers["last-modified"]))
                    os.utime(photo_name, (modtime, modtime))
            else:
                self.log('Skipping file', photo_name, 'as it already exists.')

            if photos_saved >= num_to_download:
                break

        self.log('Saved', photos_saved, 'files to', self.directory)

    def find_links(self):
        """
        Find all image urls/metadata in html_source.

        Returns a list of dicts.
        e.g., [{'display_src': 'http://image.url',
                'caption': 'some text from Instagram post',
                'date_time': '1448420058.0'},
               {...},]
        'display_src' must be present in each dict;
            the other keys are optional.
        """
        photos = []
        if self.use_metadata:
            if not GExiv2:
                self.use_metadata = False
                self.log('GExiv2 python module not found.',
                         'Images will not be tagged.')
        try:
            json_data = re.search(r'(?s)<script [^>]*>window\._sharedData'
                                  r'.*?"nodes".+?</script>',
                                  self.html_source)
            json_data = re.search(r'{.+}', json_data.group(0))
            json_data = json.loads(json_data.group(0))
            photos = list(gen_dict_extract('nodes', json_data))[0]
            # find profile_pic
            profile_pic = list(gen_dict_extract('profile_pic_url', json_data))
            if profile_pic:
                # todo (possible):
                #   add a key in the dict to indicate this is profile_pic.
                #   then we could also name the file "profile.jpg" or similar
                #   and also not include it in photos_saved so if user
                #   uses -n N to download some number of images, he still gets
                #   the first N images rather than N-1 images plus
                #   profile_pic
                profile_pic = [{'display_src': p} for p in profile_pic[:1]]
                photos = profile_pic + photos
        except:
            if self.use_metadata:
                self.use_metadata = False
                self.log('Could not find any image metadata.',
                         'Photos will not be tagged.')
            links = re.finditer(r'src="([https]+:...[\/\w \.-]*..[\/\w \.-]*'
                                r'..[\/\w \.-]*..[\/\w \.-].jpg)',
                                self.html_source)
            photos = [{'display_src': m.group(1)} for m in links]
        for photo in photos:
            date = photo.get('date')
            if date:
                try:
                    photo['date_time'] = datetime.fromtimestamp(date)
                except:
                    photo['date_time'] = None
        return photos

    def download_videos(self):
        """
        Given source code for loaded Instagram page:
        - discover all video wrapper links
        - activate all links to load video url
        - extract and download video url
        """

        if not self.get_videos:
            return;

        # We need to use the driver to query the video wrappers
        driver = self.webdriver

        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        videos_saved = 0
        self.log("Saving videos to", self.directory)

        # Find all of the video wrappers
        video_wrapper_elements = driver.find_elements_by_xpath('.//*[@id="react-root"]/section/main/article/div/div[1]/div/a[.//*[@Class="w79 f99"]]')
        video_wrapper_urls = [link.get_attribute('href') for link in video_wrapper_elements]

        for video_wrapper in video_wrapper_urls:
            # Fetch the link of the video wrapper
            driver.get(video_wrapper)

            # Wait until the real video appears
            WebDriverWait(driver, 60).until(
                expected_conditions.presence_of_all_elements_located(
                    (By.CLASS_NAME, 's68')
                )
            )

            # Get the real video, since only 1 video can be clicked on at a time, we only expect there to be a single result
            video_elements = driver.find_elements_by_class_name('s68')
            if len(video_elements) > 0:
                video_url = video_elements[0].get_attribute('src')
                video_name = op.join(self.directory, video_url.split('/')[len(video_url.split('/')) - 1])

                if not op.isfile(video_name):
                    self.save_photo(video_url, video_name)
                    videos_saved += 1
                    self.log('Downloaded file {}/{} ({}).'.format(
                        videos_saved, num_to_download, op.basename(video_name)))
                else:
                    self.log('Skipping file', video_name, 'as it already exists.')

                if videos_saved >= num_to_download:
                    break

        self.log('Saved', videos_saved, 'videos to', self.directory)

    def add_metadata(self, photo_name, caption, date_time):
        """
        Tag downloaded photos with metadata from associated Instagram post.

        If GExiv2 is not installed, do nothing.
        """
        if GExiv2:
            if caption or date_time:
                # todo: improve error handling
                try:
                    exif = GExiv2.Metadata(photo_name)
                    if caption:
                        try:
                            exif.set_comment(caption)
                        except:
                            self.log('Error setting image caption metadata.')
                    if date_time:
                        try:
                            exif.set_date_time(date_time)
                        except:
                            self.log('Error setting image date metadata.')
                    exif.save_file()
                except:
                    pass

def gen_dict_extract(key, var):
    """
    Recursively search for given dict key in nested dicts.

    from http://stackoverflow.com/a/29652561
    author: hexerei software
    """
    if hasattr(var,'iteritems'):
        for k, v in var.iteritems():
            if k == key:
                yield v
            if isinstance(v, dict):
                for result in gen_dict_extract(key, v):
                    yield result
            elif isinstance(v, list):
                for d in v:
                    for result in gen_dict_extract(key, d):
                        yield result

def main():
    # parse arguments
    parser = argparse.ArgumentParser(description='InstaRaider')
    parser.add_argument('username', help='Instagram username')
    parser.add_argument('directory', help='Where to save the images')
    parser.add_argument('-n', '--num-to-download',
                        help='Number of posts to download', type=int)
    parser.add_argument('-l', '--log-level', help="Log level", default='info')
    parser.add_argument('-m', '--add_metadata',
                        help=("Add metadata (caption/date) from Instagram "
                              "post into downloaded images' exif tags "
                              "(requires GExiv2 python module)"),
                        action='store_true', dest='use_metadata')
    parser.add_argument('-v', '--get_videos',
                        help=("Download videos"),
                        action='store_true', dest='get_videos')
    args = parser.parse_args()
    username = args.username
    directory = op.expanduser(args.directory)

    raider = InstaRaider(username, directory,
                         num_to_download=args.num_to_download,
                         log_level=args.log_level,
                         use_metadata=args.use_metadata,
                         get_videos=args.get_videos)

    if not raider.validate():
        return

    raider.download_photos()
    raider.download_videos()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import json
from models import Project
from flask import Blueprint, abort, redirect, render_template, request, url_for

app = Blueprint('projects', __name__,
                template_folder='templates')

@app.route('/projects')
def index():
    projects = Project.get_all()
    return render_template('index.html', projects=projects)

@app.route('/projects', methods=['POST'])
def create():
    name = request.form['name'].strip()
    if (name == ''): abort(400)

    url = request.form['url'].strip()
    if (url == ''): abort(400)

    desc = request.form['desc'].strip()

    project = Project(name=name, url=url, desc=desc)
    project.put()

    redirect('/projects/' + str(project.key.id()))

@app.route('/projects/<int:project_id>', methods=['GET'])
def show(project_id):
    project = Project.get_by_id(long(project_id))
    if (project == None):
        return abort(404)
    else:
        pjson = json.dumps(project.json(), ensure_ascii=False)
        return render_template('show.html', project=project, project_json=pjson)

@app.route('/projects/<int:project_id>', methods=['PATCH'])
def update(project_id):
    abort(503)

@app.route('/projects/<int:project_id>', methods=['DELETE'])
def destroy(project_id):
    abort(503)<commit_msg>return a redirect<commit_after>import json
from models import Project
from flask import Blueprint, abort, redirect, render_template, request, url_for

app = Blueprint('projects', __name__,
                template_folder='templates')

@app.route('/projects')
def index():
    projects = Project.get_all()
    return render_template('index.html', projects=projects)

@app.route('/projects', methods=['POST'])
def create():
    name = request.form['name'].strip()
    if (name == ''): abort(400)

    url = request.form['url'].strip()
    if (url == ''): abort(400)

    desc = request.form['desc'].strip()

    project = Project(name=name, url=url, desc=desc)
    project.put()

    return redirect('/projects/' + str(project.key.id()))

@app.route('/projects/<int:project_id>', methods=['GET'])
def show(project_id):
    project = Project.get_by_id(long(project_id))
    if (project == None):
        return abort(404)
    else:
        pjson = json.dumps(project.json(), ensure_ascii=False)
        return render_template('show.html', project=project, project_json=pjson)

@app.route('/projects/<int:project_id>', methods=['PATCH'])
def update(project_id):
    abort(503)

@app.route('/projects/<int:project_id>', methods=['DELETE'])
def destroy(project_id):
    abort(503)<|endoftext|>


<commit_before>import logging
import multiprocessing
import re
import os
import tempfile
import yaml
from collections import OrderedDict

# Adapted from: https://github.com/pnnl/atlas/blob/master/atlas/conf.py

# http://stackoverflow.com/a/3675423
def replace_last(source_string, replace_what, replace_with):
    head, _sep, tail =  source_string.rpartition(replace_what)
    if _sep == '':
        return tail
    else:
        return head + replace_with + tail

def get_sample_files(path):
    samples = OrderedDict()
    seen = set()
    for dir_name, sub_dirs, files in os.walk(path):
        for fname in files:

            if ".fastq" in fname or ".fq" in fname:

                sample_id = fname.partition(".fastq")[0]
                if ".fq" in sample_id:
                    sample_id = fname.partition(".fq")[0]

                sample_id = sample_id.replace("_R1", "").replace("_r1", "").replace("_R2", "").replace("_r2", "")
                sample_id = re.sub("_1$", "", sample_id)
                sample_id = re.sub("_2$", "", sample_id)
#                sample_id = replace_last(sample_id, "_1", "")
#                sample_id = replace_last(sample_id, "_2", "")
                sample_id = sample_id.replace("_", "-").replace(" ", "-")
#                print(sample_id)

                fq_path = os.path.join(dir_name, fname)
                fastq_paths = [fq_path]

                if fq_path in seen: continue

                print(fname)
                if "_R1" in fname or "_r1" in fname or "_1" in fname:
                    fname = replace_last(fname,"_1.","_2.")
                    r2_path = os.path.join(dir_name, fname.replace("_R1", "_R2").replace("_r1", "_r2"))
                    print(r2_path)
                    if not r2_path == fq_path:
                        seen.add(r2_path)
                        fastq_paths.append(r2_path)

                if "_R2" in fname or "_r2" in fname or "_2" in fname:
                    fname = replace_last(fname,"_2.","_1.")
                    r1_path = os.path.join(dir_name, fname.replace("_R2", "_R1").replace("_r2", "_r1"))
                    if not r1_path == fq_path:
                        seen.add(r1_path)
                        fastq_paths.insert(0, r1_path)

                if sample_id in samples:
                    logging.warn("Duplicate sample %s was found after renaming; skipping..." % sample_id)
                    continue
                samples[sample_id] = {'path': fastq_paths }
    return samples

def make_config(config, path):
    """Write the file `config` and complete the sample names and paths for all files in `path`."""
    represent_dict_order = lambda self, data:  self.represent_mapping('tag:yaml.org,2002:map', data.items())
    yaml.add_representer(OrderedDict, represent_dict_order)
    path = os.path.realpath(path)

    conf = OrderedDict()
    samples = get_sample_files(path)

    logging.info("Found %d samples under %s" % (len(samples), path))
    conf["project"] = "My-Project"
    conf["adapters_fasta"] = "/data/ngs/adapters/contaminant_list.txt"
    conf["pandaseq_overlap"] = "10"
    conf["pandaseq_quality"] = "25"
    conf["pandaseq_minlength"] = "100"
    conf["pandaseq_maxlength"] = "700"

    conf["forward_primer"] = "CCTACGGGNGGCWGCAG"
    conf["reverse_primer"] = "GACTACHVGGGTATCTAATCC"

    conf["silva_arb"] = "/data/db/Silva/128/SSURef_NR99_128_SILVA_07_09_16_opt.arb"
    conf["metadata"] = "data/metadata.txt"

    conf["data"] = samples

    with open(config, "w") as f:
        print(yaml.dump(conf, default_flow_style=False), file=f)
    logging.info("Configuration file written to %s" % config)

if __name__ == "__main__":
    make_config(config="config.yaml", path="data")

<commit_msg>cleanup<commit_after>import logging
import multiprocessing
import re
import os
import tempfile
import yaml
from collections import OrderedDict

# Adapted from: https://github.com/pnnl/atlas/blob/master/atlas/conf.py

# http://stackoverflow.com/a/3675423
def replace_last(source_string, replace_what, replace_with):
    head, _sep, tail =  source_string.rpartition(replace_what)
    if _sep == '':
        return tail
    else:
        return head + replace_with + tail

def get_sample_files(path):
    samples = OrderedDict()
    seen = set()
    for dir_name, sub_dirs, files in os.walk(path):
        for fname in files:

            if ".fastq" in fname or ".fq" in fname:

                sample_id = fname.partition(".fastq")[0]
                if ".fq" in sample_id:
                    sample_id = fname.partition(".fq")[0]

                sample_id = sample_id.replace("_R1", "").replace("_r1", "").replace("_R2", "").replace("_r2", "")
                sample_id = re.sub("_1$", "", sample_id)
                sample_id = re.sub("_2$", "", sample_id)
                sample_id = sample_id.replace("_", "-").replace(" ", "-")

                fq_path = os.path.join(dir_name, fname)
                fastq_paths = [fq_path]

                if fq_path in seen: continue

                if "_R1" in fname or "_r1" in fname or "_1" in fname:
                    fname = replace_last(fname,"_1.","_2.")
                    r2_path = os.path.join(dir_name, fname.replace("_R1", "_R2").replace("_r1", "_r2"))
                    if not r2_path == fq_path:
                        seen.add(r2_path)
                        fastq_paths.append(r2_path)

                if "_R2" in fname or "_r2" in fname or "_2" in fname:
                    fname = replace_last(fname,"_2.","_1.")
                    r1_path = os.path.join(dir_name, fname.replace("_R2", "_R1").replace("_r2", "_r1"))
                    if not r1_path == fq_path:
                        seen.add(r1_path)
                        fastq_paths.insert(0, r1_path)

                if sample_id in samples:
                    logging.warn("Duplicate sample %s was found after renaming; skipping..." % sample_id)
                    continue
                samples[sample_id] = {'path': fastq_paths }
    return samples

def make_config(config, path):
    """Write the file `config` and complete the sample names and paths for all files in `path`."""
    represent_dict_order = lambda self, data:  self.represent_mapping('tag:yaml.org,2002:map', data.items())
    yaml.add_representer(OrderedDict, represent_dict_order)
    path = os.path.realpath(path)

    conf = OrderedDict()
    samples = get_sample_files(path)

    logging.info("Found %d samples under %s" % (len(samples), path))
    conf["project"] = "My-Project"
    conf["adapters_fasta"] = "/data/ngs/adapters/contaminant_list.txt"
    conf["pandaseq_overlap"] = "10"
    conf["pandaseq_quality"] = "25"
    conf["pandaseq_minlength"] = "100"
    conf["pandaseq_maxlength"] = "700"

    conf["forward_primer"] = "CCTACGGGNGGCWGCAG"
    conf["reverse_primer"] = "GACTACHVGGGTATCTAATCC"

    conf["silva_arb"] = "/data/db/Silva/128/SSURef_NR99_128_SILVA_07_09_16_opt.arb"
    conf["metadata"] = "data/metadata.txt"

    conf["data"] = samples

    with open(config, "w") as f:
        print(yaml.dump(conf, default_flow_style=False), file=f)
    logging.info("Configuration file written to %s" % config)

if __name__ == "__main__":
    make_config(config="config.yaml", path="data")

<|endoftext|>