<commit_before>"""
Stores details about the available payment options.
Also stores credit card info in an encrypted format.
"""

from Crypto.Cipher import Blowfish
from datetime import datetime
from decimal import Decimal
from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from livesettings import config_value, config_choice_values, SettingNotSet
from payment.fields import PaymentChoiceCharField, CreditChoiceCharField
from satchmo_store.contact.models import Contact
from satchmo_store.shop.models import OrderPayment
import base64
import config
import keyedcache
import logging
import satchmo_utils.sslurllib

log = logging.getLogger('payment.models')

class PaymentOption(models.Model):
    """
    If there are multiple options - CC, Cash, COD, etc this class allows
    configuration.
    """
    description = models.CharField(_("Description"), max_length=20)
    active = models.BooleanField(_("Active"),
        help_text=_("Should this be displayed as an option for the user?"))
    optionName = PaymentChoiceCharField(_("Option Name"), max_length=20,
        unique=True,
        help_text=_("The class name as defined in payment.py"))
    sortOrder = models.IntegerField(_("Sort Order"))

    class Meta:
        verbose_name = _("Payment Option")
        verbose_name_plural = _("Payment Options")

class CreditCardDetail(models.Model):
    """
    Stores an encrypted CC number, its information, and its
    displayable number.
    """
    orderpayment = models.ForeignKey(OrderPayment, unique=True,
        related_name="creditcards")
    credit_type = CreditChoiceCharField(_("Credit Card Type"), max_length=16)
    display_cc = models.CharField(_("CC Number (Last 4 digits)"),
        max_length=4, )
    encrypted_cc = models.CharField(_("Encrypted Credit Card"),
        max_length=40, blank=True, null=True, editable=False)
    expire_month = models.IntegerField(_("Expiration Month"))
    expire_year = models.IntegerField(_("Expiration Year"))
    card_holder = models.CharField(_("card_holder Name"), max_length=60, blank=True)
    start_month = models.IntegerField(_("Start Month"), blank=True, null=True)
    start_year = models.IntegerField(_("Start Year"), blank=True, null=True)
    issue_num = models.CharField(blank=True, null=True, max_length=2)

    def storeCC(self, ccnum):
        """Take as input a valid cc, encrypt it and store the last 4 digits in a visible form"""
        self.display_cc = ccnum[-4:]
        encrypted_cc = _encrypt_code(ccnum)
        if config_value('PAYMENT', 'STORE_CREDIT_NUMBERS'):
            self.encrypted_cc = encrypted_cc
        else:
            standin = "%s%i%i%i" % (self.display_cc, self.expire_month, self.expire_year, self.orderpayment.id)
            self.encrypted_cc = _encrypt_code(standin)
            key = _encrypt_code(standin + '-card')
            keyedcache.cache_set(key, skiplog=True, length=60*60, value=encrypted_cc)

    def setCCV(self, ccv):
        """Put the CCV in the cache, don't save it for security/legal reasons."""
        if not self.encrypted_cc:
            raise ValueError('CreditCardDetail expecting a credit card number to be stored before storing CCV')

        keyedcache.cache_set(self.encrypted_cc, skiplog=True, length=60*60, value=ccv)

    def getCCV(self):
        try:
            ccv = keyedcache.cache_get(self.encrypted_cc)
        except keyedcache.NotCachedError:
            ccv = ""

        return ccv

    ccv = property(fget=getCCV, fset=setCCV)

    def _decryptCC(self):
        ccnum = _decrypt_code(self.encrypted_cc)
        if not config_value('PAYMENT', 'STORE_CREDIT_NUMBERS'):
            try:
                key = _encrypt_code(ccnum + '-card')
                encrypted_ccnum = keyedcache.cache_get(key)
                ccnum = _decrypt_code(encrypted_ccnum)
            except keyedcache.NotCachedError:
                ccnum = ""
        return ccnum

    decryptedCC = property(_decryptCC)

    def _expireDate(self):
        return(str(self.expire_month) + "/" + str(self.expire_year))
    expirationDate = property(_expireDate)

    class Meta:
        verbose_name = _("Credit Card")
        verbose_name_plural = _("Credit Cards")

def _decrypt_code(code):
    """Decrypt code encrypted by _encrypt_code"""
    secret_key = settings.SECRET_KEY
    encryption_object = Blowfish.new(secret_key)
    # strip padding from decrypted credit card number
    return encryption_object.decrypt(base64.b64decode(code)).rstrip('X')

def _encrypt_code(code):
    """Quick encrypter for CC codes or code fragments"""
    secret_key = settings.SECRET_KEY
    encryption_object = Blowfish.new(secret_key)
    # block cipher length must be a multiple of 8
    padding = ''
    if (len(code) % 8) <> 0:
        padding = 'X' * (8 - (len(code) % 8))
    return base64.b64encode(encryption_object.encrypt(code + padding))
<commit_msg>Merging.<commit_after>"""
Stores details about the available payment options.
Also stores credit card info in an encrypted format.
"""

from Crypto.Cipher import Blowfish
from datetime import datetime
from decimal import Decimal
from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from livesettings import config_value, config_choice_values, SettingNotSet
from payment.fields import PaymentChoiceCharField, CreditChoiceCharField
from satchmo_store.contact.models import Contact
import base64
import config
import keyedcache
import logging
import satchmo_utils.sslurllib

log = logging.getLogger('payment.models')

class PaymentOption(models.Model):
    """
    If there are multiple options - CC, Cash, COD, etc this class allows
    configuration.
    """
    description = models.CharField(_("Description"), max_length=20)
    active = models.BooleanField(_("Active"),
        help_text=_("Should this be displayed as an option for the user?"))
    optionName = PaymentChoiceCharField(_("Option Name"), max_length=20,
        unique=True,
        help_text=_("The class name as defined in payment.py"))
    sortOrder = models.IntegerField(_("Sort Order"))

    class Meta:
        verbose_name = _("Payment Option")
        verbose_name_plural = _("Payment Options")

class CreditCardDetail(models.Model):
    """
    Stores an encrypted CC number, its information, and its
    displayable number.
    """
    orderpayment = models.ForeignKey('shop.OrderPayment', unique=True,
        related_name="creditcards")
    credit_type = CreditChoiceCharField(_("Credit Card Type"), max_length=16)
    display_cc = models.CharField(_("CC Number (Last 4 digits)"),
        max_length=4, )
    encrypted_cc = models.CharField(_("Encrypted Credit Card"),
        max_length=40, blank=True, null=True, editable=False)
    expire_month = models.IntegerField(_("Expiration Month"))
    expire_year = models.IntegerField(_("Expiration Year"))
    card_holder = models.CharField(_("card_holder Name"), max_length=60, blank=True)
    start_month = models.IntegerField(_("Start Month"), blank=True, null=True)
    start_year = models.IntegerField(_("Start Year"), blank=True, null=True)
    issue_num = models.CharField(blank=True, null=True, max_length=2)

    def storeCC(self, ccnum):
        """Take as input a valid cc, encrypt it and store the last 4 digits in a visible form"""
        self.display_cc = ccnum[-4:]
        encrypted_cc = _encrypt_code(ccnum)
        if config_value('PAYMENT', 'STORE_CREDIT_NUMBERS'):
            self.encrypted_cc = encrypted_cc
        else:
            standin = "%s%i%i%i" % (self.display_cc, self.expire_month, self.expire_year, self.orderpayment.id)
            self.encrypted_cc = _encrypt_code(standin)
            key = _encrypt_code(standin + '-card')
            keyedcache.cache_set(key, skiplog=True, length=60*60, value=encrypted_cc)

    def setCCV(self, ccv):
        """Put the CCV in the cache, don't save it for security/legal reasons."""
        if not self.encrypted_cc:
            raise ValueError('CreditCardDetail expecting a credit card number to be stored before storing CCV')

        keyedcache.cache_set(self.encrypted_cc, skiplog=True, length=60*60, value=ccv)

    def getCCV(self):
        try:
            ccv = keyedcache.cache_get(self.encrypted_cc)
        except keyedcache.NotCachedError:
            ccv = ""

        return ccv

    ccv = property(fget=getCCV, fset=setCCV)

    def _decryptCC(self):
        ccnum = _decrypt_code(self.encrypted_cc)
        if not config_value('PAYMENT', 'STORE_CREDIT_NUMBERS'):
            try:
                key = _encrypt_code(ccnum + '-card')
                encrypted_ccnum = keyedcache.cache_get(key)
                ccnum = _decrypt_code(encrypted_ccnum)
            except keyedcache.NotCachedError:
                ccnum = ""
        return ccnum

    decryptedCC = property(_decryptCC)

    def _expireDate(self):
        return(str(self.expire_month) + "/" + str(self.expire_year))
    expirationDate = property(_expireDate)

    class Meta:
        verbose_name = _("Credit Card")
        verbose_name_plural = _("Credit Cards")

def _decrypt_code(code):
    """Decrypt code encrypted by _encrypt_code"""
    secret_key = settings.SECRET_KEY
    encryption_object = Blowfish.new(secret_key)
    # strip padding from decrypted credit card number
    return encryption_object.decrypt(base64.b64decode(code)).rstrip('X')

def _encrypt_code(code):
    """Quick encrypter for CC codes or code fragments"""
    secret_key = settings.SECRET_KEY
    encryption_object = Blowfish.new(secret_key)
    # block cipher length must be a multiple of 8
    padding = ''
    if (len(code) % 8) <> 0:
        padding = 'X' * (8 - (len(code) % 8))
    return base64.b64encode(encryption_object.encrypt(code + padding))
<|endoftext|>


<commit_before>import os,subprocess
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.fileutils import get_temp_dir, remove_dir
from igf_data.utils.fileutils import copy_remote_file
from igf_data.utils.projectutils import get_project_read_count,get_seqrun_info_for_project
from igf_data.utils.gviz_utils import convert_to_gviz_json_for_display
from igf_data.utils.project_data_display_utils import convert_project_data_gviz_data,add_seqrun_path_info
from igf_data.utils.project_status_utils import Project_status

class UpdateProjectInfo(IGFBaseProcess):
  '''
  An ehive runnable class for updating data for project info page
  '''
  def param_defaults(self):
    params_dict=super(UpdateProjectInfo,self).param_defaults()
    params_dict.update({
      'remote_project_path':None,
      'remote_user':None,
      'remote_host':None,
      'seqruninfofile':'seqruninfofile.json',
      'samplereadcountfile':'samplereadcountfile.json',
      'status_data_json':'status_data.json',
      'pipeline_name':None,
    })
    return params_dict

  def run(self):
    try:
      seqrun_igf_id=self.param_required('seqrun_igf_id')
      project_name=self.param_required('project_name')
      remote_project_path=self.param_required('remote_project_path')
      igf_session_class=self.param_required('igf_session_class')
      remote_user=self.param_required('remote_user')
      remote_host=self.param_required('remote_host')
      seqruninfofile=self.param('seqruninfofile')
      samplereadcountfile=self.param('samplereadcountfile')
      status_data_json=self.param('status_data_json')
      pipeline_name=self.param('pipeline_name')

      temp_work_dir=get_temp_dir()                                              # get a temp dir
      temp_read_count_output=os.path.join(temp_work_dir,
                                          samplereadcountfile)                  # get path for temp read count file
      temp_seqrun_info=os.path.join(temp_work_dir,
                                    seqruninfofile)                             # get path for temp seqrun info file
      raw_read_count=get_project_read_count(session_class=igf_session_class,
                                            project_igf_id=project_name)        # get raw read count for project
      (description,read_count_data,column_order)=\
              convert_project_data_gviz_data(input_data=raw_read_count)         # convert read count to gviz requirements
      convert_to_gviz_json_for_display(description=description,
                                       data=read_count_data,
                                       columns_order=column_order,
                                       output_file=temp_read_count_output)      # write data to output json file
      seqrun_data=get_seqrun_info_for_project(session_class=igf_session_class,
                                            project_igf_id=project_name)        # fetch seqrun info for each projects
      add_seqrun_path_info(input_data=seqrun_data,
                           output_file=temp_seqrun_info)                        # write seqrun info json
      remote_project_dir=os.path.join(remote_project_path,\
                                      project_name)                             # get remote project directory path
      check_seqrun_cmd=['ssh',\
                        '{0}@{1}'.\
                        format(remote_user,\
                               remote_host),\
                        'ls',\
                        os.path.join(remote_project_dir,seqruninfofile)]
      response=subprocess.call(check_seqrun_cmd)                                # check for existing seqruninfofile
      if response !=0:
        rm_seqrun_cmd=['ssh',\
                       '{0}@{1}'.\
                       format(remote_user,\
                              remote_host),\
                       'rm',\
                       '-f',\
                       os.path.join(remote_project_dir,
                                    seqruninfofile)]
        subprocess.check_call(rm_seqrun_cmd)                                    # remove existing seqruninfofile
      os.chmod(temp_seqrun_info, mode=0o754)                                    # changed file permission before copy
      copy_remote_file(source_path=temp_seqrun_info, \
                       destinationa_path=remote_project_dir, \
                       destination_address=remote_host)                         # copy file to remote
      check_readcount_cmd=['ssh',\
                           '{0}@{1}'.\
                           format(remote_user,\
                                  remote_host),\
                           'ls',\
                           os.path.join(remote_project_dir,
                                        samplereadcountfile)]                   # check for existing samplereadcountfile
      response=subprocess.call(check_readcount_cmd)
      if response !=0:
        rm_readcount_cmd=['ssh',\
                          '{0}@{1}'.\
                          format(remote_user,\
                                 remote_host),\
                          'rm',\
                          '-f',\
                          os.path.join(remote_project_dir,
                                       samplereadcountfile)]
        subprocess.check_call(rm_readcount_cmd)                                 # remove existing samplereadcountfile
      os.chmod(temp_read_count_output, mode=0o754)                              # changed file permission before copy
      copy_remote_file(source_path=temp_read_count_output, \
                       destinationa_path=remote_project_dir, \
                       destination_address=remote_host)                         # copy file to remote

      ps=Project_status(igf_session_class=igf_session_class,
                        project_igf_id=project_name)
      temp_status_output=os.path.join(temp_work_dir,
                                      status_data_json)                         # get path for temp status file
      convert_to_gviz_json_for_display(\
        description=ps.get_status_description(),
        data=ps.get_seqrun_info(demultiplexing_pipeline=pipeline_name,
                                active_seqrun_igf_id=seqrun_igf_id),
        columns_order=ps.get_status_column_order(),
        output_file=temp_status_output)                                         # write data to output json file
      os.chmod(temp_status_output,
               mode=0o754)                                                      # changed file permission before copy
      check_status_cmd=['ssh',\
                        '{0}@{1}'.\
                        format(remote_user,\
                               remote_host),\
                        'ls',\
                        os.path.join(remote_project_dir,
                                     status_data_json)]
      response=subprocess.call(check_status_cmd)                                # check for existing status data file
      if response !=0:
        rm_status_cmd=['ssh',\
                       '{0}@{1}'.\
                       format(remote_user,\
                              remote_host),\
                       'rm',\
                       '-f',\
                       os.path.join(remote_project_dir,
                                    status_data_json)]
        subprocess.check_call(rm_status_cmd)                                    # remove existing status file

      copy_remote_file(source_path=temp_status_output, \
                       destinationa_path=remote_project_dir, \
                       destination_address=remote_host)                         # copy file to remote
      self.param('dataflow_params',{'remote_project_info':'done'})
      remove_dir(temp_work_dir)                                                 # remove temp dir
    except Exception as e:
      message='seqrun: {2}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      seqrun_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<commit_msg>updated project info runnable for updated grantt plot module<commit_after>import os,subprocess
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.fileutils import get_temp_dir, remove_dir
from igf_data.utils.fileutils import copy_remote_file
from igf_data.utils.projectutils import get_project_read_count,get_seqrun_info_for_project
from igf_data.utils.gviz_utils import convert_to_gviz_json_for_display
from igf_data.utils.project_data_display_utils import convert_project_data_gviz_data,add_seqrun_path_info
from igf_data.utils.project_status_utils import Project_status

class UpdateProjectInfo(IGFBaseProcess):
  '''
  An ehive runnable class for updating data for project info page
  '''
  def param_defaults(self):
    params_dict=super(UpdateProjectInfo,self).param_defaults()
    params_dict.update({
      'remote_project_path':None,
      'remote_user':None,
      'remote_host':None,
      'seqruninfofile':'seqruninfofile.json',
      'samplereadcountfile':'samplereadcountfile.json',
      'status_data_json':'status_data.json',
      'pipeline_name':None,
    })
    return params_dict

  def run(self):
    try:
      seqrun_igf_id=self.param_required('seqrun_igf_id')
      project_name=self.param_required('project_name')
      remote_project_path=self.param_required('remote_project_path')
      igf_session_class=self.param_required('igf_session_class')
      remote_user=self.param_required('remote_user')
      remote_host=self.param_required('remote_host')
      seqruninfofile=self.param('seqruninfofile')
      samplereadcountfile=self.param('samplereadcountfile')
      status_data_json=self.param('status_data_json')
      pipeline_name=self.param_required('pipeline_name')
      analysis_pipeline_name=self.param_required('analysis_pipeline_name')

      temp_work_dir=get_temp_dir()                                              # get a temp dir
      temp_read_count_output=os.path.join(temp_work_dir,
                                          samplereadcountfile)                  # get path for temp read count file
      temp_seqrun_info=os.path.join(temp_work_dir,
                                    seqruninfofile)                             # get path for temp seqrun info file
      raw_read_count=get_project_read_count(session_class=igf_session_class,
                                            project_igf_id=project_name)        # get raw read count for project
      (description,read_count_data,column_order)=\
              convert_project_data_gviz_data(input_data=raw_read_count)         # convert read count to gviz requirements
      convert_to_gviz_json_for_display(description=description,
                                       data=read_count_data,
                                       columns_order=column_order,
                                       output_file=temp_read_count_output)      # write data to output json file
      seqrun_data=get_seqrun_info_for_project(session_class=igf_session_class,
                                            project_igf_id=project_name)        # fetch seqrun info for each projects
      add_seqrun_path_info(input_data=seqrun_data,
                           output_file=temp_seqrun_info)                        # write seqrun info json
      remote_project_dir=os.path.join(remote_project_path,\
                                      project_name)                             # get remote project directory path
      check_seqrun_cmd=['ssh',\
                        '{0}@{1}'.\
                        format(remote_user,\
                               remote_host),\
                        'ls',\
                        os.path.join(remote_project_dir,seqruninfofile)]
      response=subprocess.call(check_seqrun_cmd)                                # check for existing seqruninfofile
      if response !=0:
        rm_seqrun_cmd=['ssh',\
                       '{0}@{1}'.\
                       format(remote_user,\
                              remote_host),\
                       'rm',\
                       '-f',\
                       os.path.join(remote_project_dir,
                                    seqruninfofile)]
        subprocess.check_call(rm_seqrun_cmd)                                    # remove existing seqruninfofile
      os.chmod(temp_seqrun_info, mode=0o754)                                    # changed file permission before copy
      copy_remote_file(source_path=temp_seqrun_info, \
                       destinationa_path=remote_project_dir, \
                       destination_address=remote_host)                         # copy file to remote
      check_readcount_cmd=['ssh',\
                           '{0}@{1}'.\
                           format(remote_user,\
                                  remote_host),\
                           'ls',\
                           os.path.join(remote_project_dir,
                                        samplereadcountfile)]                   # check for existing samplereadcountfile
      response=subprocess.call(check_readcount_cmd)
      if response !=0:
        rm_readcount_cmd=['ssh',\
                          '{0}@{1}'.\
                          format(remote_user,\
                                 remote_host),\
                          'rm',\
                          '-f',\
                          os.path.join(remote_project_dir,
                                       samplereadcountfile)]
        subprocess.check_call(rm_readcount_cmd)                                 # remove existing samplereadcountfile
      os.chmod(temp_read_count_output, mode=0o754)                              # changed file permission before copy
      copy_remote_file(source_path=temp_read_count_output, \
                       destinationa_path=remote_project_dir, \
                       destination_address=remote_host)                         # copy file to remote

      ps=Project_status(igf_session_class=igf_session_class,
                        project_igf_id=project_name)
      temp_status_output=os.path.join(temp_work_dir,
                                      status_data_json)                         # get path for temp status file
      ps.generate_gviz_json_file(\
           output_file=temp_status_output,
           demultiplexing_pipeline=pipeline_name,
           analysis_pipeline=analysis_pipeline_name,
           active_seqrun_igf_id=seqrun_igf_id)                                  # write data to output json file
      os.chmod(temp_status_output,
               mode=0o754)                                                      # changed file permission before copy
      check_status_cmd=['ssh',\
                        '{0}@{1}'.\
                        format(remote_user,\
                               remote_host),\
                        'ls',\
                        os.path.join(remote_project_dir,
                                     status_data_json)]
      response=subprocess.call(check_status_cmd)                                # check for existing status data file
      if response !=0:
        rm_status_cmd=['ssh',\
                       '{0}@{1}'.\
                       format(remote_user,\
                              remote_host),\
                       'rm',\
                       '-f',\
                       os.path.join(remote_project_dir,
                                    status_data_json)]
        subprocess.check_call(rm_status_cmd)                                    # remove existing status file

      copy_remote_file(source_path=temp_status_output, \
                       destinationa_path=remote_project_dir, \
                       destination_address=remote_host)                         # copy file to remote
      self.param('dataflow_params',{'remote_project_info':'done'})
      remove_dir(temp_work_dir)                                                 # remove temp dir
    except Exception as e:
      message='seqrun: {2}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      seqrun_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<|endoftext|>


<commit_before>import asyncio
import json
import logging
import os
import shlex
import re

from . import process_provider
from .. import exception
from xml.dom import minidom

logger = logging.getLogger(__name__)

# TODO: NCL-3503: Finish implementation once the other components are figured out
def get_pme_provider(execution_name, pme_jar_path, pme_parameters, output_to_logs=False, specific_indy_group=None, timestamp=None):
    @asyncio.coroutine
    def get_result_data(work_dir):
        raw_result_data = "{}"
        result_file_path = work_dir + "/target/pom-manip-ext-result.json"
        if os.path.isfile(result_file_path):
            with open(result_file_path, "r") as file:
                raw_result_data = file.read()
        logger.info('Got PME result data "{raw_result_data}".'.format(**locals()))
        pme_result = json.loads(raw_result_data)
        try:
            pme_result["RemovedRepositories"] = get_removed_repos(work_dir, pme_parameters)
        except FileNotFoundError as e:
            logger.error('File for removed repositories could not be found')
            logger.error(str(e))
        return pme_result

    def get_removed_repos(work_dir, parameters):
        """
        Parses the filename of the removed repos backup file from the parameters list and if there
        is one, it reads the list of repos and returns it.
        """
        result = []

        pattern = re.compile("-DrepoRemovalBackup[ =](.+)")
        for parameter in parameters:
            m = pattern.match(parameter)
            if m is not None:
                filepath = os.path.join(work_dir, m.group(1))
                logger.debug('Files and folders in the work directory:\n  %s', os.listdir(work_dir))

                if os.path.exists(filepath):
                    tree = minidom.parse(filepath)
                    for repo_elem in tree.getElementsByTagName("repository"):
                        repo = {"releases": True, "snapshots": True, "name": "", "id": "", "url": ""}
                        for enabled_elem in repo_elem.getElementsByTagName("enabled"):
                            if enabled_elem.parentNode.localName in ["releases", "snapshots"]:
                                bool_value = enabled_elem.childNodes[0].data == "true"
                                repo[enabled_elem.parentNode.localName] = bool_value
                        for tag in ["id", "name", "url"]:
                            for elem in repo_elem.getElementsByTagName(tag):
                                repo[tag] = elem.childNodes[0].data
                        result.append(repo)
                    break
                else:
                    logger.info('File %s does not exist. It seems no repositories were removed '
                                'by PME.', filepath)

        return result

    @asyncio.coroutine
    def get_extra_parameters(extra_adjust_parameters):
        paramsString = extra_adjust_parameters.get("CUSTOM_PME_PARAMETERS", None)
        if paramsString is None:
            return []
        else:
            params = shlex.split(paramsString)
            for p in params:
                if p[0] != "-":
                    desc = ('Parameters that do not start with dash "-" are not allowed. '
                            + 'Found "{p}" in "{params}".'.format(**locals()))
                    raise exception.AdjustCommandError(desc, [], 10, stderr=desc)
            return params

    @asyncio.coroutine
    def adjust(repo_dir, extra_adjust_parameters, adjust_result):
        nonlocal execution_name

        temp_build_parameters = []
        if timestamp:
            temp_build_parameters.append("-DversionIncrementalSuffix=" + timestamp + "-redhat")

        if specific_indy_group:
            temp_build_parameters.append("-DrestRepositoryGroup=" + specific_indy_group)

        cmd = ["java", "-jar", pme_jar_path] \
              + pme_parameters \
              + temp_build_parameters \
              + (yield from get_extra_parameters(extra_adjust_parameters))
        logger.info('Executing "' + execution_name + '" using "pme" adjust provider '
                    + '(delegating to "process" provider). Command is "{cmd}".'.format(**locals()))
        res = yield from process_provider.get_process_provider(execution_name,
                                                     cmd,
                                                     get_result_data=get_result_data,
                                                     send_log=output_to_logs) \
            (repo_dir, extra_adjust_parameters, adjust_result)
        adjust_result['resultData'] = yield from get_result_data(repo_dir)
        return res

    return adjust

@asyncio.coroutine
def get_version_from_pme_result(pme_result):
    """
    Format of pme_result should be as follows:

    {
      "VersioningState": {
        "executionRootModified": {
          "groupId": "<group-id>",
          "artifactId": "<artifact-id>",
          "version": "<pme'd version>"
        }
      }
    }

    Function tries to extract version generated by PME from the pme_result

    Parameters:
    - pme_result: :dict:
    """
    try:
        version = pme_result['VersioningState']['executionRootModified']['version']
        return version
    except  Exception as e:
        logger.error("Couldn't extract PME result version from JSON file")
        logger.error(e)
        return None<commit_msg>[NCL-3601] Correctly process non top-level pom.xml<commit_after>import asyncio
import json
import logging
import os
import shlex
import re

from . import process_provider
from .. import exception
from xml.dom import minidom

logger = logging.getLogger(__name__)

# TODO: NCL-3503: Finish implementation once the other components are figured out
def get_pme_provider(execution_name, pme_jar_path, pme_parameters, output_to_logs=False, specific_indy_group=None, timestamp=None):
    @asyncio.coroutine
    def get_result_data(work_dir):
        raw_result_data = "{}"
        result_file_path = work_dir + "/target/pom-manip-ext-result.json"
        if os.path.isfile(result_file_path):
            with open(result_file_path, "r") as file:
                raw_result_data = file.read()
        logger.info('Got PME result data "{raw_result_data}".'.format(**locals()))
        pme_result = json.loads(raw_result_data)
        try:
            pme_result["RemovedRepositories"] = get_removed_repos(work_dir, pme_parameters)
        except FileNotFoundError as e:
            logger.error('File for removed repositories could not be found')
            logger.error(str(e))
        return pme_result

    def get_removed_repos(work_dir, parameters):
        """
        Parses the filename of the removed repos backup file from the parameters list and if there
        is one, it reads the list of repos and returns it.
        """
        result = []

        pattern = re.compile("-DrepoRemovalBackup[ =](.+)")
        for parameter in parameters:
            m = pattern.match(parameter)
            if m is not None:
                filepath = os.path.join(work_dir, m.group(1))
                logger.debug('Files and folders in the work directory:\n  %s', os.listdir(work_dir))

                if os.path.exists(filepath):
                    tree = minidom.parse(filepath)
                    for repo_elem in tree.getElementsByTagName("repository"):
                        repo = {"releases": True, "snapshots": True, "name": "", "id": "", "url": ""}
                        for enabled_elem in repo_elem.getElementsByTagName("enabled"):
                            if enabled_elem.parentNode.localName in ["releases", "snapshots"]:
                                bool_value = enabled_elem.childNodes[0].data == "true"
                                repo[enabled_elem.parentNode.localName] = bool_value
                        for tag in ["id", "name", "url"]:
                            for elem in repo_elem.getElementsByTagName(tag):
                                repo[tag] = elem.childNodes[0].data
                        result.append(repo)
                    break
                else:
                    logger.info('File %s does not exist. It seems no repositories were removed '
                                'by PME.', filepath)

        return result

    @asyncio.coroutine
    def get_extra_parameters(extra_adjust_parameters):
        """
        Get the extra PME parameters from PNC
        If the PME parameters contain '--file=<folder>/pom.xml', then extract that folder
        and remove that --file option from the list of extra params.
        In PME 2.11 and PME 2.12, there is a bug where that option causes the file target/pom-manip-ext-result.json
        to be badly generated. Fixed in PME 2.13+
        See: PRODTASKS-361
        Returns: tuple<list<string>, string>: list of params(minus the --file option), and folder where to run PME
        If '--file' option not used, the folder will be an empty string
        """
        subfolder = ''

        paramsString = extra_adjust_parameters.get("CUSTOM_PME_PARAMETERS", None)
        if paramsString is None:
            return [], subfolder
        else:
            params = shlex.split(paramsString)
            for p in params:
                if p[0] != "-":
                    desc = ('Parameters that do not start with dash "-" are not allowed. '
                            + 'Found "{p}" in "{params}".'.format(**locals()))
                    raise exception.AdjustCommandError(desc, [], 10, stderr=desc)
                if p.startswith("--file"):
                    subfolder = p.replace("--file=", "").replace("pom.xml", "")

            params_without_file_option = [p for p in params if not p.startswith("--file=")]

            return params_without_file_option, subfolder

    @asyncio.coroutine
    def adjust(repo_dir, extra_adjust_parameters, adjust_result):
        nonlocal execution_name

        temp_build_parameters = []
        if timestamp:
            temp_build_parameters.append("-DversionIncrementalSuffix=" + timestamp + "-redhat")

        if specific_indy_group:
            temp_build_parameters.append("-DrestRepositoryGroup=" + specific_indy_group)

        extra_parameters, subfolder = yield from get_extra_parameters(extra_adjust_parameters)

        # readjust the repo_dir to run PME from the folder where the root pom.xml is located
        # See: PRODTASKS-361
        repo_dir = os.path.join(repo_dir, subfolder)

        cmd = ["java", "-jar", pme_jar_path] \
              + pme_parameters + temp_build_parameters + extra_parameters

        logger.info('Executing "' + execution_name + '" using "pme" adjust provider '
                    + '(delegating to "process" provider). Command is "{cmd}".'.format(**locals()))
        res = yield from process_provider.get_process_provider(execution_name,
                                                     cmd,
                                                     get_result_data=get_result_data,
                                                     send_log=output_to_logs) \
            (repo_dir, extra_adjust_parameters, adjust_result)
        adjust_result['resultData'] = yield from get_result_data(repo_dir)
        return res

    return adjust

@asyncio.coroutine
def get_version_from_pme_result(pme_result):
    """
    Format of pme_result should be as follows:

    {
      "VersioningState": {
        "executionRootModified": {
          "groupId": "<group-id>",
          "artifactId": "<artifact-id>",
          "version": "<pme'd version>"
        }
      }
    }

    Function tries to extract version generated by PME from the pme_result

    Parameters:
    - pme_result: :dict:
    """
    try:
        version = pme_result['VersioningState']['executionRootModified']['version']
        return version
    except  Exception as e:
        logger.error("Couldn't extract PME result version from JSON file")
        logger.error(e)
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
import wx
import windbreads.utils as wdu
import wxbreads.utils as wxu
import wxbreads.images as wxi
import wxbreads.widgets as wxw

class BaseWindow(wx.Frame):
    clock_timer_id = wx.NewId()
    echo_timer_id = wx.NewId()
    root_pass = 'guess'
    app_version = '0.1'
    app_name = 'Cool app'
    app_remark = 'Description for cool app'
    app_author = 'Cool Author'

    def __init__(self, **kwargs):
        super(BaseWindow, self).__init__(kwargs.get('parent'),
                                         size=kwargs.get('size', (-1, -1)),
                                         title=self.app_name)
        self.is_running = False
        self.echo_lines = []
        self.is_echoing = False
        self.t = None
        self.has_tray = False
        self.logo = img = wxi.logo.GetImage()
        icon = wx.IconFromBitmap(img.ConvertToBitmap())
        self.SetIcon(icon)
        self.copyright = kwargs.get('copyright') or wdu.get_copy_right()
        self.setup_statusbar()
        self.Bind(wx.EVT_CLOSE, self.on_quit)

    def show_window(self):
        self.Centre(wx.BOTH)
        self.Show()

    def setup_timers(self, clock_timer=1000, echo_timer=200):
        self.clock_timer = wxw.init_timer(self, self.clock_timer_id,
                                          self.on_clock_tick, clock_timer)
        self.echo_timer = wxw.init_timer(self, self.echo_timer_id,
                                         self.on_echoing, echo_timer)
        self.all_timers = [self.clock_timer, self.echo_timer]

    def stop_timers(self):
        wxu.stop_timers(self.all_timers)

    def on_clock_tick(self, evt=None):
        wxu.update_clock_statusbar(self.sbar, idx=len(self.get_sb_width() - 1))

    def setup_statusbar(self):
        self.sbar = wxw.init_statusbar(self, widths=self.get_sb_width(),
                                       values=self.get_sb_value())

    def get_sb_width(self):
        """Width items for status bar."""
        return [230, -1, 120]

    def get_sb_value(self):
        """Value items for status bar."""
        return [self.copyright, '', '']

    def update_status(self, text, idx, **kwargs):
        kwargs.setdefault('t', self.t)
        wxu.set_status_text(self.sbar, text, idx, **kwargs)

    def setup_tray(self, **kwargs):
        import wxbreads.trayicon as wxt
        try:
            args = dict(icon=kwargs.get('icon') or self.logo,
                        text=kwargs.get('text', 'Tray'),
                        t=self.t)
            self.tbicon = wxt.TrayIcon(self, **args)
            self.opened_dlg = 0
            self.has_tray = True
        except:
            self.tbicon = None

    def update_t(self):
        wdu.update_t(self, lang=self.get_lang(), zh=self.get_zh_mo(),
                     en=self.get_en_mo())

    def get_lang(self):
        return 'en'

    def get_zh_mo(self):
        return None

    def get_en_mo(self):
        return None

    def tt(self, text):
        return wdu.ttt(text, self.t)

    def on_quit(self, evt=None):
        wxw.quick_quit(self, t=self.t)

    def on_hide(self, evt=None):
        self.Hide()

    def echo_text(self, text='', **kwargs):
        kwargs.setdefault('t', self.t)
        kwargs.setdefault('log_mode', 'a')
        kwargs.setdefault('log_files', [])
        wxu.echo_text(self.rtc, text, **kwargs)

    def add_echo(self, text='', **kwargs):
        self.echo_lines.append((text, kwargs))

    def on_echoing(self, evt=None):
        wxu.on_echoing(self)

    def popup(self, caption, msg, icon='i', **kwargs):
        if self.has_tray:
            self.opened_dlg += 1
            if kwargs.pop('restore', True):
                self.tbicon.on_restore(None)

        kwargs.setdefault('t', self.t)
        result = wxw.popup(caption=caption, msg=msg, icon=icon, **kwargs)
        if self.has_tray:
            self.opened_dlg -= 1

        return result

    def on_setting(self, evt=None):
        if self.prepare_setting():
            self.open_setting()

    def open_setting(self):
        raise NotImplementedError

    def prepare_setting(self):
        if self.is_running:
            self.popup('Warning', 'Please stop current running task', 'w')
            return False

        self.add_echo('Updating settings...', fg='blue', italic=True)
        if self.has_tray:
            self.opened_dlg += 1

        if not wxw.quick_password_entry(self, root_pass=self.root_pass,
                                        t=self.t):
            if self.has_tray:
                self.opened_dlg -= 1

            return False

        return True

    def on_about(self, evt=None):
        if self.has_tray:
            self.opened_dlg += 1

        wxw.quick_about(name=self.app_name,
                        version=self.app_version,
                        author=self.app_author,
                        icon=wxi.phoenix.getIcon(),
                        remark=self.app_remark,
                        t=self.t)
        if self.has_tray:
            self.opened_dlg -= 1

    def on_popup_lang(self, evt):
        wxu.on_popup_lang(self, evt)

    def save_lang(self, lang=None):
        raise NotImplementedError

    def popup_lang(self, evt):
        wxu.popup_lang(self, evt)

    def update_ui_lang(self, refresh=True):
        wxu.update_ui_lang(self, refresh)
<commit_msg>copyright is from method<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
import wx
import windbreads.utils as wdu
import wxbreads.utils as wxu
import wxbreads.images as wxi
import wxbreads.widgets as wxw

class BaseWindow(wx.Frame):
    clock_timer_id = wx.NewId()
    echo_timer_id = wx.NewId()
    root_pass = 'guess'
    app_version = '0.1'
    app_name = 'Cool app'
    app_remark = 'Description for cool app'
    app_author = 'Cool Author'

    def __init__(self, **kwargs):
        super(BaseWindow, self).__init__(kwargs.get('parent'),
                                         size=kwargs.get('size', (-1, -1)),
                                         title=self.app_name)
        self.is_running = False
        self.echo_lines = []
        self.is_echoing = False
        self.t = None
        self.has_tray = False
        self.logo = img = wxi.logo.GetImage()
        icon = wx.IconFromBitmap(img.ConvertToBitmap())
        self.SetIcon(icon)
        self.setup_statusbar()
        self.Bind(wx.EVT_CLOSE, self.on_quit)

    def show_window(self):
        self.Centre(wx.BOTH)
        self.Show()

    def setup_timers(self, clock_timer=1000, echo_timer=200):
        self.clock_timer = wxw.init_timer(self, self.clock_timer_id,
                                          self.on_clock_tick, clock_timer)
        self.echo_timer = wxw.init_timer(self, self.echo_timer_id,
                                         self.on_echoing, echo_timer)
        self.all_timers = [self.clock_timer, self.echo_timer]

    def stop_timers(self):
        wxu.stop_timers(self.all_timers)

    def on_clock_tick(self, evt=None):
        wxu.update_clock_statusbar(self.sbar, idx=len(self.get_sb_width() - 1))

    def setup_statusbar(self):
        self.sbar = wxw.init_statusbar(self, widths=self.get_sb_width(),
                                       values=self.get_sb_value())

    def get_sb_width(self):
        """Width items for status bar."""
        return [230, -1, 120]

    def get_sb_value(self):
        """Value items for status bar."""
        return [self.get_copyright(), '', '']

    def get_copyright(self):
        return wdu.get_copy_right()

    def update_status(self, text, idx, **kwargs):
        kwargs.setdefault('t', self.t)
        wxu.set_status_text(self.sbar, text, idx, **kwargs)

    def setup_tray(self, **kwargs):
        import wxbreads.trayicon as wxt
        try:
            args = dict(icon=kwargs.get('icon') or self.logo,
                        text=kwargs.get('text', 'Tray'),
                        t=self.t)
            self.tbicon = wxt.TrayIcon(self, **args)
            self.opened_dlg = 0
            self.has_tray = True
        except:
            self.tbicon = None

    def update_t(self):
        wdu.update_t(self, lang=self.get_lang(), zh=self.get_zh_mo(),
                     en=self.get_en_mo())

    def get_lang(self):
        return 'en'

    def get_zh_mo(self):
        return None

    def get_en_mo(self):
        return None

    def tt(self, text):
        return wdu.ttt(text, self.t)

    def on_quit(self, evt=None):
        wxw.quick_quit(self, t=self.t)

    def on_hide(self, evt=None):
        self.Hide()

    def echo_text(self, text='', **kwargs):
        kwargs.setdefault('t', self.t)
        kwargs.setdefault('log_mode', 'a')
        kwargs.setdefault('log_files', [])
        wxu.echo_text(self.rtc, text, **kwargs)

    def add_echo(self, text='', **kwargs):
        self.echo_lines.append((text, kwargs))

    def on_echoing(self, evt=None):
        wxu.on_echoing(self)

    def popup(self, caption, msg, icon='i', **kwargs):
        if self.has_tray:
            self.opened_dlg += 1
            if kwargs.pop('restore', True):
                self.tbicon.on_restore(None)

        kwargs.setdefault('t', self.t)
        result = wxw.popup(caption=caption, msg=msg, icon=icon, **kwargs)
        if self.has_tray:
            self.opened_dlg -= 1

        return result

    def on_setting(self, evt=None):
        if self.prepare_setting():
            self.open_setting()

    def open_setting(self):
        raise NotImplementedError

    def prepare_setting(self):
        if self.is_running:
            self.popup('Warning', 'Please stop current running task', 'w')
            return False

        self.add_echo('Updating settings...', fg='blue', italic=True)
        if self.has_tray:
            self.opened_dlg += 1

        if not wxw.quick_password_entry(self, root_pass=self.root_pass,
                                        t=self.t):
            if self.has_tray:
                self.opened_dlg -= 1

            return False

        return True

    def on_about(self, evt=None):
        if self.has_tray:
            self.opened_dlg += 1

        wxw.quick_about(name=self.app_name,
                        version=self.app_version,
                        author=self.app_author,
                        icon=wxi.phoenix.getIcon(),
                        remark=self.app_remark,
                        t=self.t)
        if self.has_tray:
            self.opened_dlg -= 1

    def on_popup_lang(self, evt):
        wxu.on_popup_lang(self, evt)

    def save_lang(self, lang=None):
        raise NotImplementedError

    def popup_lang(self, evt):
        wxu.popup_lang(self, evt)

    def update_ui_lang(self, refresh=True):
        wxu.update_ui_lang(self, refresh)
<|endoftext|>


<commit_before>"""
A #pathlib.Path implementation for URLs.
"""

from nodepy.utils import pathlib
import os
import io
import posixpath

try:
  from urllib.request import urlopen
  from urllib.parse import urlparse, urlunparse
except ImportError:
  from urllib2 import urlopen
  from urlparse import urlparse, urlunparse

class _UrlFlavour(pathlib._PosixFlavour):
  sep = '/'
  altsep = ''
  has_drv = False
  pathmod = posixpath
  is_supported = True

  def splitroot(self, part, sep=sep):
    res = urlparse(part)
    return (
      res.scheme + '://' if res.scheme else '',
      res.netloc + '/' if res.netloc else '',
      urlunparse(('', '', res.path, res.params, res.query, res.fragment))
    )

class PureUrlPath(pathlib.PurePath):
  _flavour = _UrlFlavour()
  __slots__ = ()

class UrlPath(pathlib.Path, PureUrlPath):
  __slots__ = ()

  # Wrapper for the socket._fileobject returned from #urlopen().
  # Necessary in Python 2 because socket._fileobject does not support
  # readable(), writable() and seekable(), and without this protocol it
  # can not be wrapped in #io.BufferedReader or #io.TextIOWrapper.
  class _readable(object):
    def __init__(self, fp, seekable=False):
      self._fp = fp
      self._seekable = seekable
      self._closed = False
    def __getattr__(self, name):
      return getattr(self._fp, name)
    def readable(self):
      return True
    def writable(self):
      return False
    def seekable(self):
      return self._seekable

  def owner(self):
    raise NotImplementedError("Path.owner() is unsupported for URLs")

  def group(self):
    raise NotImplementedError("Path.group() is unsupported for URLs")

  def open(self, flags='r', mode=0o666):
    if set(flags).difference('rbt'):
      raise IOError('URLs can be opened in read-mode only.')
    if six.PY2:
      fp = self._readable(urlopen(str(self)).fp)
    else:
      fp = urlopen(str(self))
    if not isinstance(fp, io.BufferedReader):
      fp = io.BufferedReader(fp)
    if 'b' not in flags:
      fp = io.TextIOWrapper(fp)
    return fp

  def is_dir(self):
    return False

  def is_file(self):
    return False

  def is_symlink(self):
    return False

  def is_socket(self):
    return False

  def is_fifo(self):
    return False

  def is_char_device(self):
    return False

  def is_block_device(self):
    return False
<commit_msg>add missing six import to urlpath<commit_after>"""
A #pathlib.Path implementation for URLs.
"""

from nodepy.utils import pathlib
import os
import io
import posixpath
import six

try:
  from urllib.request import urlopen
  from urllib.parse import urlparse, urlunparse
except ImportError:
  from urllib2 import urlopen
  from urlparse import urlparse, urlunparse

class _UrlFlavour(pathlib._PosixFlavour):
  sep = '/'
  altsep = ''
  has_drv = False
  pathmod = posixpath
  is_supported = True

  def splitroot(self, part, sep=sep):
    res = urlparse(part)
    return (
      res.scheme + '://' if res.scheme else '',
      res.netloc + '/' if res.netloc else '',
      urlunparse(('', '', res.path, res.params, res.query, res.fragment))
    )

class PureUrlPath(pathlib.PurePath):
  _flavour = _UrlFlavour()
  __slots__ = ()

class UrlPath(pathlib.Path, PureUrlPath):
  __slots__ = ()

  # Wrapper for the socket._fileobject returned from #urlopen().
  # Necessary in Python 2 because socket._fileobject does not support
  # readable(), writable() and seekable(), and without this protocol it
  # can not be wrapped in #io.BufferedReader or #io.TextIOWrapper.
  class _readable(object):
    def __init__(self, fp, seekable=False):
      self._fp = fp
      self._seekable = seekable
      self._closed = False
    def __getattr__(self, name):
      return getattr(self._fp, name)
    def readable(self):
      return True
    def writable(self):
      return False
    def seekable(self):
      return self._seekable

  def owner(self):
    raise NotImplementedError("Path.owner() is unsupported for URLs")

  def group(self):
    raise NotImplementedError("Path.group() is unsupported for URLs")

  def open(self, flags='r', mode=0o666):
    if set(flags).difference('rbt'):
      raise IOError('URLs can be opened in read-mode only.')
    if six.PY2:
      fp = self._readable(urlopen(str(self)).fp)
    else:
      fp = urlopen(str(self))
    if not isinstance(fp, io.BufferedReader):
      fp = io.BufferedReader(fp)
    if 'b' not in flags:
      fp = io.TextIOWrapper(fp)
    return fp

  def is_dir(self):
    return False

  def is_file(self):
    return False

  def is_symlink(self):
    return False

  def is_socket(self):
    return False

  def is_fifo(self):
    return False

  def is_char_device(self):
    return False

  def is_block_device(self):
    return False
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
insta_raider.py

usage: insta_raider.py [-h] -u USERNAME

@amirkurtovic

"""
import argparse
import logging
import os
import os.path as op
import re
import email.utils as eut
import requests
import calendar
try:
    from requests.packages.urllib3.exceptions import InsecurePlatformWarning
except ImportError:
    from urllib3.exceptions import InsecurePlatformWarning
import time
import warnings
import selenium.webdriver as webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import NoSuchElementException
import json
from datetime import datetime
try:
    from gi.repository import GExiv2
except ImportError:
    GExiv2 = None

warnings.filterwarnings("ignore", category=InsecurePlatformWarning)

class PrivateUserError(Exception):
    """Raised if the profile is found to be private"""

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

class InstaRaider(object):

    def __init__(self, username, directory, num_to_download=None,
                 log_level='info', use_metadata=False, get_videos=False):
        self.username = username
        self.profile_url = self.get_url(username)
        self.directory = directory
        self.PAUSE = 1
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        self.headers = {'User-Agent': self.user_agent}
        self.html_source = None
        self.log_level = getattr(logging, log_level.upper())
        self.setup_logging(self.log_level)
        self.use_metadata = use_metadata
        self.get_videos = get_videos
        self.set_num_posts(num_to_download)
        self.setup_webdriver()

    def __del__(self):
        if self.webdriver:
            self.webdriver.close()

    def get_url(self, path):
        return urlparse.urljoin('https://instagram.com', path)

    def set_num_posts(self, num_to_download=None):
        self.num_posts = int(self.get_posts_count(self.profile_url) or 0)
        self.num_to_download = num_to_download

    def setup_logging(self, level=logging.INFO):
        self.logger = logging.getLogger('instaraider')
        self.logger.addHandler(logging.StreamHandler())
        self.logger.setLevel(logging.INFO)

    def log(self, *strings, **kwargs):
        level = kwargs.pop('level', logging.INFO)
        self.logger.log(level, u' '.join(str(s) for s in strings))

    def setup_webdriver(self):
        self.profile = webdriver.FirefoxProfile()
        self.profile.set_preference("general.useragent.override", self.user_agent)
        self.webdriver = webdriver.Firefox(self.profile)
        self.webdriver.set_window_size(480, 320)
        self.webdriver.set_window_position(800, 0)

    def get_posts_count(self, url):
        """
        Given a url to Instagram profile, return number of photos posted
        """
        response = requests.get(url)
        counts_code = re.search(r'\"media":{"count":\d+', response.text)
        if not counts_code:
            return None
        return re.findall(r'\d+', counts_code.group())[0]

    def log_in_user(self):
        driver = self.webdriver
        self.log('You need to login to access this profile.',
                 'Redirecting you to the login page in the browser.',
                 level=logging.WARN)
        driver.get(self.get_url('accounts/login/'))

        # Wait until user has been successfully logged in and redirceted
        # to his/her feed.
        WebDriverWait(driver, 60).until(
            expected_conditions.presence_of_element_located(
                (By.CSS_SELECTOR, '.-cx-PRIVATE-FeedPage__feed'),
            )
        )

        self.log('User successfully logged in.', level=logging.INFO)
        self.set_num_posts()  # Have to set this again
        driver.get(self.profile_url)

    def load_instagram(self):
        """
        Using Selenium WebDriver, load Instagram page to get page source

        """
        self.log(self.username, 'has', self.num_posts, 'posts on Instagram.')
        if self.num_to_download is not None:
            self.log("The first", self.num_to_download, "of them will be downloaded.")

        num_to_download = self.num_to_download or self.num_posts
        driver = self.webdriver
        # load Instagram profile and wait for PAUSE
        self.log("Loading Instagram profile...")
        driver.get(self.profile_url)
        driver.implicitly_wait(self.PAUSE)

        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        try:
            el = driver.find_element_by_css_selector(
                '.-cx-PRIVATE-ProfilePage__advisoryMessageHeader'
            )
        except NoSuchElementException:
            pass
        else:
            self.log_in_user()

        if (num_to_download > 24):
            scroll_to_bottom = self.get_scroll_count(num_to_download)
            element = driver.find_element_by_css_selector('a.i09')
            driver.implicitly_wait(self.PAUSE)
            element.click()

            for y in range(int(scroll_to_bottom)):
                self.scroll_page(driver)

        # After load all profile photos, retur, source to download_photos()
        time.sleep(1)
        source = driver.page_source

        return source

    def scroll_page(self, driver):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(0.2)
        driver.execute_script("window.scrollTo(0, 0);")

    def get_scroll_count(self, count):
        return (int(count) - 24) / 12 + 1

    def validate(self):
        """
        returns True if Instagram username is valid
        """
        req = requests.get(self.profile_url)

        try:
            req.raise_for_status()
        except:
            self.log('User', self.username, 'is not valid.',
                     level=logging.ERROR)
            return False

        if not self.num_posts:
            self.log('User', self.username, 'has no photos to download.',
                     level=logging.ERROR)
            return False
        return True

    def save_photo(self, photo_url, photo_name):
        image_request = requests.get(photo_url, headers=self.headers)
        image_data = image_request.content
        with open(photo_name, 'wb') as fp:
            fp.write(image_data)
        return image_request.headers

    def download_photos(self):
        """
        Given source code for loaded Instagram page,
        extract all hrefs and download full-resolution photos

        source: HTML source code of Instagram profile papge
        """
        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()

        # check if directory exists, if not, make it
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        # index for progress bar
        photos_saved = 0
        self.log("Saving photos to", self.directory)

        links = self.find_links()

        for link in links:
            photo_url = link['display_src']
            photo_url = photo_url.replace('\\', '')
            photo_url = re.sub(r'/s\d+x\d+/', '/', photo_url)
            photo_url = re.sub(r'/\w+\.\d+\.\d+\.\d+/', '/', photo_url)
            caption = link.get('caption')
            date_time = link.get('date_time')
            photo_basename = op.basename(photo_url)
            photo_name = op.join(self.directory, photo_basename)

            # save full-resolution photo if its new
            if not op.isfile(photo_name):
                headers = self.save_photo(photo_url, photo_name)
                photos_saved += 1
                self.log('Downloaded file {}/{} ({}).'.format(
                    photos_saved, num_to_download, photo_basename))
                # put info from Instagram post into image metadata
                if self.use_metadata:
                    self.add_metadata(photo_name, caption, date_time)
                if "last-modified" in headers:
                    modtime = calendar.timegm(eut.parsedate(headers["last-modified"]))
                    os.utime(photo_name, (modtime, modtime))
            else:
                self.log('Skipping file', photo_name, 'as it already exists.')

            if photos_saved >= num_to_download:
                break

        self.log('Saved', photos_saved, 'files to', self.directory)

    def find_links(self):
        """
        Find all image urls/metadata in html_source.

        Returns a list of dicts.
        e.g., [{'display_src': 'http://image.url',
                'caption': 'some text from Instagram post',
                'date_time': '1448420058.0'},
               {...},]
        'display_src' must be present in each dict;
            the other keys are optional.
        """
        photos = []
        if self.use_metadata:
            if not GExiv2:
                self.use_metadata = False
                self.log('GExiv2 python module not found.',
                         'Images will not be tagged.')
        try:
            json_data = re.search(r'(?s)<script [^>]*>window\._sharedData'
                                  r'.*?"nodes".+?</script>',
                                  self.html_source)
            json_data = re.search(r'{.+}', json_data.group(0))
            json_data = json.loads(json_data.group(0))
            photos = list(gen_dict_extract('nodes', json_data))[0]
            # find profile_pic
            profile_pic = list(gen_dict_extract('profile_pic_url', json_data))
            if profile_pic:
                # todo (possible):
                #   add a key in the dict to indicate this is profile_pic.
                #   then we could also name the file "profile.jpg" or similar
                #   and also not include it in photos_saved so if user
                #   uses -n N to download some number of images, he still gets
                #   the first N images rather than N-1 images plus
                #   profile_pic
                profile_pic = [{'display_src': p} for p in profile_pic[:1]]
                photos = profile_pic + photos
        except:
            if self.use_metadata:
                self.use_metadata = False
                self.log('Could not find any image metadata.',
                         'Photos will not be tagged.')
            links = re.finditer(r'src="([https]+:...[\/\w \.-]*..[\/\w \.-]*'
                                r'..[\/\w \.-]*..[\/\w \.-].jpg)',
                                self.html_source)
            photos = [{'display_src': m.group(1)} for m in links]
        for photo in photos:
            date = photo.get('date')
            if date:
                try:
                    photo['date_time'] = datetime.fromtimestamp(date)
                except:
                    photo['date_time'] = None
        return photos

    def download_videos(self):
        """
        Given source code for loaded Instagram page:
        - discover all video wrapper links
        - activate all links to load video url
        - extract and download video url
        """

        if not self.get_videos:
            return;

        # We need to use the driver to query the video wrappers
        driver = self.webdriver

        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        videos_saved = 0
        self.log("Saving videos to", self.directory)

        # Find all of the video wrappers
        video_wrapper_elements = driver.find_elements_by_xpath('.//*[@id="react-root"]/section/main/article/div/div[1]/div/a[.//*[@Class="w79 f99"]]')
        video_wrapper_urls = [link.get_attribute('href') for link in video_wrapper_elements]

        for video_wrapper in video_wrapper_urls:
            # Fetch the link of the video wrapper
            driver.get(video_wrapper)

            # Wait until the real video appears
            WebDriverWait(driver, 60).until(
                expected_conditions.presence_of_all_elements_located(
                    (By.CLASS_NAME, 's68')
                )
            )

            # Get the real video, since only 1 video can be clicked on at a time, we only expect there to be a single result
            video_elements = driver.find_elements_by_class_name('s68')
            if len(video_elements) > 0:
                video_url = video_elements[0].get_attribute('src')
                video_name = op.join(self.directory, video_url.split('/')[len(video_url.split('/')) - 1])

                if not op.isfile(video_name):
                    self.save_photo(video_url, video_name)
                    videos_saved += 1
                    self.log('Downloaded file {}/{} ({}).'.format(
                        videos_saved, num_to_download, op.basename(video_name)))
                else:
                    self.log('Skipping file', video_name, 'as it already exists.')

                if videos_saved >= num_to_download:
                    break

        self.log('Saved', videos_saved, 'videos to', self.directory)

    def add_metadata(self, photo_name, caption, date_time):
        """
        Tag downloaded photos with metadata from associated Instagram post.

        If GExiv2 is not installed, do nothing.
        """
        if GExiv2:
            if caption or date_time:
                # todo: improve error handling
                try:
                    exif = GExiv2.Metadata(photo_name)
                    if caption:
                        try:
                            exif.set_comment(caption)
                        except:
                            self.log('Error setting image caption metadata.')
                    if date_time:
                        try:
                            exif.set_date_time(date_time)
                        except:
                            self.log('Error setting image date metadata.')
                    exif.save_file()
                except:
                    pass

def gen_dict_extract(key, var):
    """
    Recursively search for given dict key in nested dicts.

    from http://stackoverflow.com/a/29652561
    author: hexerei software
    """
    if hasattr(var,'iteritems'):
        for k, v in var.iteritems():
            if k == key:
                yield v
            if isinstance(v, dict):
                for result in gen_dict_extract(key, v):
                    yield result
            elif isinstance(v, list):
                for d in v:
                    for result in gen_dict_extract(key, d):
                        yield result

def main():
    # parse arguments
    parser = argparse.ArgumentParser(description='InstaRaider')
    parser.add_argument('username', help='Instagram username')
    parser.add_argument('directory', help='Where to save the images')
    parser.add_argument('-n', '--num-to-download',
                        help='Number of posts to download', type=int)
    parser.add_argument('-l', '--log-level', help="Log level", default='info')
    parser.add_argument('-m', '--add_metadata',
                        help=("Add metadata (caption/date) from Instagram "
                              "post into downloaded images' exif tags "
                              "(requires GExiv2 python module)"),
                        action='store_true', dest='use_metadata')
    parser.add_argument('-v', '--get_videos',
                        help=("Download videos"),
                        action='store_true', dest='get_videos')
    args = parser.parse_args()
    username = args.username
    directory = op.expanduser(args.directory)

    raider = InstaRaider(username, directory,
                         num_to_download=args.num_to_download,
                         log_level=args.log_level,
                         use_metadata=args.use_metadata,
                         get_videos=args.get_videos)

    if not raider.validate():
        return

    raider.download_photos()
    raider.download_videos()

if __name__ == '__main__':
    main()
<commit_msg>Fixes 'LOAD MORE' link error + only attempts to load JSON data if metadata use is attempted.<commit_after>#!/usr/bin/env python
"""
insta_raider.py

usage: insta_raider.py [-h] -u USERNAME

@amirkurtovic

"""
import argparse
import logging
import os
import os.path as op
import re
import email.utils as eut
import requests
import calendar
try:
    from requests.packages.urllib3.exceptions import InsecurePlatformWarning
except ImportError:
    from urllib3.exceptions import InsecurePlatformWarning
import time
import warnings
import selenium.webdriver as webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import NoSuchElementException
import json
from datetime import datetime
try:
    from gi.repository import GExiv2
except ImportError:
    GExiv2 = None

warnings.filterwarnings("ignore", category=InsecurePlatformWarning)

class PrivateUserError(Exception):
    """Raised if the profile is found to be private"""

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

class InstaRaider(object):

    def __init__(self, username, directory, num_to_download=None,
                 log_level='info', use_metadata=False, get_videos=False):
        self.username = username
        self.profile_url = self.get_url(username)
        self.directory = directory
        self.PAUSE = 1
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        self.headers = {'User-Agent': self.user_agent}
        self.html_source = None
        self.log_level = getattr(logging, log_level.upper())
        self.setup_logging(self.log_level)
        self.use_metadata = use_metadata
        self.get_videos = get_videos
        self.set_num_posts(num_to_download)
        self.setup_webdriver()

    def __del__(self):
        if self.webdriver:
            self.webdriver.close()

    def get_url(self, path):
        return urlparse.urljoin('https://instagram.com', path)

    def set_num_posts(self, num_to_download=None):
        self.num_posts = int(self.get_posts_count(self.profile_url) or 0)
        self.num_to_download = num_to_download

    def setup_logging(self, level=logging.INFO):
        self.logger = logging.getLogger('instaraider')
        self.logger.addHandler(logging.StreamHandler())
        self.logger.setLevel(logging.INFO)

    def log(self, *strings, **kwargs):
        level = kwargs.pop('level', logging.INFO)
        self.logger.log(level, u' '.join(str(s) for s in strings))

    def setup_webdriver(self):
        self.profile = webdriver.FirefoxProfile()
        self.profile.set_preference("general.useragent.override", self.user_agent)
        self.webdriver = webdriver.Firefox(self.profile)
        self.webdriver.set_window_size(480, 320)
        self.webdriver.set_window_position(800, 0)

    def get_posts_count(self, url):
        """
        Given a url to Instagram profile, return number of photos posted
        """
        response = requests.get(url)
        counts_code = re.search(r'\"media":{"count":\d+', response.text)
        if not counts_code:
            return None
        return re.findall(r'\d+', counts_code.group())[0]

    def log_in_user(self):
        driver = self.webdriver
        self.log('You need to login to access this profile.',
                 'Redirecting you to the login page in the browser.',
                 level=logging.WARN)
        driver.get(self.get_url('accounts/login/'))

        # Wait until user has been successfully logged in and redirceted
        # to his/her feed.
        WebDriverWait(driver, 60).until(
            expected_conditions.presence_of_element_located(
                (By.CSS_SELECTOR, '.-cx-PRIVATE-FeedPage__feed'),
            )
        )

        self.log('User successfully logged in.', level=logging.INFO)
        self.set_num_posts()  # Have to set this again
        driver.get(self.profile_url)

    def load_instagram(self):
        """
        Using Selenium WebDriver, load Instagram page to get page source

        """
        self.log(self.username, 'has', self.num_posts, 'posts on Instagram.')
        if self.num_to_download is not None:
            self.log("The first", self.num_to_download, "of them will be downloaded.")

        num_to_download = self.num_to_download or self.num_posts
        driver = self.webdriver
        # load Instagram profile and wait for PAUSE
        self.log("Loading Instagram profile...")
        driver.get(self.profile_url)
        driver.implicitly_wait(self.PAUSE)

        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        try:
            el = driver.find_element_by_css_selector(
                '.-cx-PRIVATE-ProfilePage__advisoryMessageHeader'
            )
        except NoSuchElementException:
            pass
        else:
            self.log_in_user()

        if (num_to_download > 24):
            scroll_to_bottom = self.get_scroll_count(num_to_download)
            element = driver.find_element_by_css_selector('a._oidfu')
            driver.implicitly_wait(self.PAUSE)
            element.click()

            for y in range(int(scroll_to_bottom)):
                self.scroll_page(driver)

        # After load all profile photos, retur, source to download_photos()
        time.sleep(1)
        source = driver.page_source

        return source

    def scroll_page(self, driver):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(0.2)
        driver.execute_script("window.scrollTo(0, 0);")

    def get_scroll_count(self, count):
        return (int(count) - 24) / 12 + 1

    def validate(self):
        """
        returns True if Instagram username is valid
        """
        req = requests.get(self.profile_url)

        try:
            req.raise_for_status()
        except:
            self.log('User', self.username, 'is not valid.',
                     level=logging.ERROR)
            return False

        if not self.num_posts:
            self.log('User', self.username, 'has no photos to download.',
                     level=logging.ERROR)
            return False
        return True

    def save_photo(self, photo_url, photo_name):
        image_request = requests.get(photo_url, headers=self.headers)
        image_data = image_request.content
        with open(photo_name, 'wb') as fp:
            fp.write(image_data)
        return image_request.headers

    def download_photos(self):
        """
        Given source code for loaded Instagram page,
        extract all hrefs and download full-resolution photos

        source: HTML source code of Instagram profile papge
        """
        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()

        # check if directory exists, if not, make it
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        # index for progress bar
        photos_saved = 0
        self.log("Saving photos to", self.directory)

        links = self.find_links()

        for link in links:
            photo_url = link['display_src']
            photo_url = photo_url.replace('\\', '')
            photo_url = re.sub(r'/s\d+x\d+/', '/', photo_url)
            photo_url = re.sub(r'/\w+\.\d+\.\d+\.\d+/', '/', photo_url)
            caption = link.get('caption')
            date_time = link.get('date_time')
            photo_basename = op.basename(photo_url)
            photo_name = op.join(self.directory, photo_basename)

            # save full-resolution photo if its new
            if not op.isfile(photo_name):
                headers = self.save_photo(photo_url, photo_name)
                photos_saved += 1
                self.log('Downloaded file {}/{} ({}).'.format(
                    photos_saved, num_to_download, photo_basename))
                # put info from Instagram post into image metadata
                if self.use_metadata:
                    self.add_metadata(photo_name, caption, date_time)
                if "last-modified" in headers:
                    modtime = calendar.timegm(eut.parsedate(headers["last-modified"]))
                    os.utime(photo_name, (modtime, modtime))
            else:
                self.log('Skipping file', photo_name, 'as it already exists.')

            if photos_saved >= num_to_download:
                break

        self.log('Saved', photos_saved, 'files to', self.directory)

    def find_links(self):
        """
        Find all image urls/metadata in html_source.

        Returns a list of dicts.
        e.g., [{'display_src': 'http://image.url',
                'caption': 'some text from Instagram post',
                'date_time': '1448420058.0'},
               {...},]
        'display_src' must be present in each dict;
            the other keys are optional.
        """
        photos = []
        if self.use_metadata:
            if not GExiv2:
                self.use_metadata = False
                self.log('GExiv2 python module not found.',
                         'Images will not be tagged.')
            try:
                json_data = re.search(r'(?s)<script [^>]*>window\._sharedData'
                                      r'.*?"nodes".+?</script>',
                                      self.html_source)
                json_data = re.search(r'{.+}', json_data.group(0))
                json_data = json.loads(json_data.group(0))
                photos = list(gen_dict_extract('nodes', json_data))[0]
                # find profile_pic
                profile_pic = list(gen_dict_extract('profile_pic_url', json_data))
                if profile_pic:
                    # todo (possible):
                    #   add a key in the dict to indicate this is profile_pic.
                    #   then we could also name the file "profile.jpg" or similar
                    #   and also not include it in photos_saved so if user
                    #   uses -n N to download some number of images, he still gets
                    #   the first N images rather than N-1 images plus
                    #   profile_pic
                    profile_pic = [{'display_src': p} for p in profile_pic[:1]]
                    photos = profile_pic + photos
            except:
                if self.use_metadata:
                    self.use_metadata = False
                    self.log('Could not find any image metadata.',
                             'Photos will not be tagged.')
        else:
            links = re.finditer(r'src="([https]+:...[\/\w \.-]*..[\/\w \.-]*'
                                r'..[\/\w \.-]*..[\/\w \.-].jpg)',
                                self.html_source)
            photos = [{'display_src': m.group(1)} for m in links]
        for photo in photos:
            date = photo.get('date')
            if date:
                try:
                    photo['date_time'] = datetime.fromtimestamp(date)
                except:
                    photo['date_time'] = None
        return photos

    def download_videos(self):
        """
        Given source code for loaded Instagram page:
        - discover all video wrapper links
        - activate all links to load video url
        - extract and download video url
        """

        if not self.get_videos:
            return;

        # We need to use the driver to query the video wrappers
        driver = self.webdriver

        num_to_download = self.num_to_download or self.num_posts
        if self.html_source is None:
            self.html_source = self.load_instagram()
        if not op.exists(self.directory):
            os.makedirs(self.directory)

        videos_saved = 0
        self.log("Saving videos to", self.directory)

        # Find all of the video wrappers
        video_wrapper_elements = driver.find_elements_by_xpath('.//*[@id="react-root"]/section/main/article/div/div[1]/div/a[.//*[@Class="w79 f99"]]')
        video_wrapper_urls = [link.get_attribute('href') for link in video_wrapper_elements]

        for video_wrapper in video_wrapper_urls:
            # Fetch the link of the video wrapper
            driver.get(video_wrapper)

            # Wait until the real video appears
            WebDriverWait(driver, 60).until(
                expected_conditions.presence_of_all_elements_located(
                    (By.CLASS_NAME, 's68')
                )
            )

            # Get the real video, since only 1 video can be clicked on at a time, we only expect there to be a single result
            video_elements = driver.find_elements_by_class_name('s68')
            if len(video_elements) > 0:
                video_url = video_elements[0].get_attribute('src')
                video_name = op.join(self.directory, video_url.split('/')[len(video_url.split('/')) - 1])

                if not op.isfile(video_name):
                    self.save_photo(video_url, video_name)
                    videos_saved += 1
                    self.log('Downloaded file {}/{} ({}).'.format(
                        videos_saved, num_to_download, op.basename(video_name)))
                else:
                    self.log('Skipping file', video_name, 'as it already exists.')

                if videos_saved >= num_to_download:
                    break

        self.log('Saved', videos_saved, 'videos to', self.directory)

    def add_metadata(self, photo_name, caption, date_time):
        """
        Tag downloaded photos with metadata from associated Instagram post.

        If GExiv2 is not installed, do nothing.
        """
        if GExiv2:
            if caption or date_time:
                # todo: improve error handling
                try:
                    exif = GExiv2.Metadata(photo_name)
                    if caption:
                        try:
                            exif.set_comment(caption)
                        except:
                            self.log('Error setting image caption metadata.')
                    if date_time:
                        try:
                            exif.set_date_time(date_time)
                        except:
                            self.log('Error setting image date metadata.')
                    exif.save_file()
                except:
                    pass

def gen_dict_extract(key, var):
    """
    Recursively search for given dict key in nested dicts.

    from http://stackoverflow.com/a/29652561
    author: hexerei software
    """
    if hasattr(var,'iteritems'):
        for k, v in var.iteritems():
            if k == key:
                yield v
            if isinstance(v, dict):
                for result in gen_dict_extract(key, v):
                    yield result
            elif isinstance(v, list):
                for d in v:
                    for result in gen_dict_extract(key, d):
                        yield result

def main():
    # parse arguments
    parser = argparse.ArgumentParser(description='InstaRaider')
    parser.add_argument('username', help='Instagram username')
    parser.add_argument('directory', help='Where to save the images')
    parser.add_argument('-n', '--num-to-download',
                        help='Number of posts to download', type=int)
    parser.add_argument('-l', '--log-level', help="Log level", default='info')
    parser.add_argument('-m', '--add_metadata',
                        help=("Add metadata (caption/date) from Instagram "
                              "post into downloaded images' exif tags "
                              "(requires GExiv2 python module)"),
                        action='store_true', dest='use_metadata')
    parser.add_argument('-v', '--get_videos',
                        help=("Download videos"),
                        action='store_true', dest='get_videos')
    args = parser.parse_args()
    username = args.username
    directory = op.expanduser(args.directory)

    raider = InstaRaider(username, directory,
                         num_to_download=args.num_to_download,
                         log_level=args.log_level,
                         use_metadata=args.use_metadata,
                         get_videos=args.get_videos)

    if not raider.validate():
        return

    raider.download_photos()
    raider.download_videos()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import socket
import os
import sys

ROOT = "./webroot/"
ADDR = ("127.0.0.1", 8000)
server = socket.socket(
    socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_IP
)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

def resolve_uri(uri):
    if "../" in uri:
        raise UserWarning("403 Forbidden")

    if os.path.isfile(uri):
        TYPE = uri.split(".")[-1].lower()
        if TYPE in ('PNG', 'JPG', 'JPEG', 'GIF'):
            TYPE = 'image/' + TYPE
        else:
            TYPE = 'text/' + TYPE
        BODY = open(uri).read()

    elif os.path.isdir(uri):
        TYPE = "text/html"
        BODY = "<!DOCTYPE HTML><title>Directory</title>
Directory
"
        for root, dirs, files in os.walk(uri):
            BODY += "
" + root + "
"
            BODY += "
"
            for f in files:
                BODY += "
" + f + "
"
            BODY += "
"
        BODY += ""

    else:
        raise IOError("404 Not Found")
    return TYPE, BODY

def parse_request(request):
    try:
        request = request.split("\r\n\r\n", 1)
        req = request[0].split("\r\n")
        for i, r in enumerate(req):
            req[i] = r.split()
        method = req[0][0].upper()
        uri = req[0][1]
        proto = req[0][2].upper()
        headers = {}
        for line in req[1:]:
            headers[line[0].upper()] = line[1:]
        if method == "GET":
            if proto == "HTTP/1.1" and "HOST:" in headers:
                return ROOT + uri
            else:
                raise SyntaxError("400 Bad Request")
        else:
            raise ValueError("405 Method Not Allowed")
    except IndexError:
        raise SyntaxError("400 Bad Request")

def response_ok(TYPE, BODY):

    return ("HTTP/1.1 200 OK\r\n"
            "Content-Type: " + TYPE + "\r\n"
            "Content-Length: " + str(sys.getsizeof(BODY)) + "\r\n"
            "\r\n" + BODY)

def response_error(e=Exception("500 Internal Server Error")):
    html = "<title>Error!</title>
Error!

It did not work
"
    return ("HTTP/1.1 " + e.message + "\r\n"
            "Content-Type: text/plain\r\n"
            "Content-Length: " + str(len(html)) + "\r\n"
            "\r\n" + html)

if __name__ == "__main__":
    server.bind(ADDR)
    server.listen(1)
    while True:
        try:
            conn, addr = server.accept()
            s, msg = "", True
            while msg:
                msg = conn.recv(1024)
                s += msg
                if len(msg) < 1024:
                    break
            print s
            try:
                uri = parse_request(s)
                TYPE, BODY = resolve_uri(uri)
                resp = response_ok(TYPE, BODY)
            except (SyntaxError, ValueError, UserWarning) as e:
                resp = response_error(e)
        except KeyboardInterrupt as e:
            break
        except Exception as e:
            # print e
            resp = response_error()

        conn.sendall(resp)
        conn.close()
<commit_msg>fixed size display in error<commit_after>import socket
import os
import sys

ROOT = "./webroot/"
ADDR = ("127.0.0.1", 8000)
server = socket.socket(
    socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_IP
)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

def resolve_uri(uri):
    if "../" in uri:
        raise UserWarning("403 Forbidden")

    if os.path.isfile(uri):
        TYPE = uri.split(".")[-1].lower()
        if TYPE in ('PNG', 'JPG', 'JPEG', 'GIF'):
            TYPE = 'image/' + TYPE
        else:
            TYPE = 'text/' + TYPE
        BODY = open(uri).read()

    elif os.path.isdir(uri):
        TYPE = "text/html"
        BODY = "<!DOCTYPE HTML><title>Directory</title>
Directory
"
        for root, dirs, files in os.walk(uri):
            BODY += "
" + root + "
"
            BODY += "
"
            for f in files:
                BODY += "
" + f + "
"
            BODY += "
"
        BODY += ""

    else:
        raise IOError("404 Not Found")
    return TYPE, BODY

def parse_request(request):
    try:
        request = request.split("\r\n\r\n", 1)
        req = request[0].split("\r\n")
        for i, r in enumerate(req):
            req[i] = r.split()
        method = req[0][0].upper()
        uri = req[0][1]
        proto = req[0][2].upper()
        headers = {}
        for line in req[1:]:
            headers[line[0].upper()] = line[1:]
        if method == "GET":
            if proto == "HTTP/1.1" and "HOST:" in headers:
                return ROOT + uri
            else:
                raise SyntaxError("400 Bad Request")
        else:
            raise ValueError("405 Method Not Allowed")
    except IndexError:
        raise SyntaxError("400 Bad Request")

def response_ok(TYPE, BODY):

    return ("HTTP/1.1 200 OK\r\n"
            "Content-Type: " + TYPE + "\r\n"
            "Content-Length: " + str(sys.getsizeof(BODY)) + "\r\n"
            "\r\n" + BODY)

def response_error(e=Exception("500 Internal Server Error")):
    html = "<title>Error!</title>
Error!

It did not work
"
    return ("HTTP/1.1 " + e.message + "\r\n"
            "Content-Type: text/plain\r\n"
            "Content-Length: " + str(sys.getsizeof(html)) + "\r\n"
            "\r\n" + html)

if __name__ == "__main__":
    server.bind(ADDR)
    server.listen(1)
    while True:
        try:
            conn, addr = server.accept()
            s, msg = "", True
            while msg:
                msg = conn.recv(1024)
                s += msg
                if len(msg) < 1024:
                    break
            print s
            try:
                uri = parse_request(s)
                TYPE, BODY = resolve_uri(uri)
                resp = response_ok(TYPE, BODY)
            except (SyntaxError, ValueError, UserWarning) as e:
                resp = response_error(e)
        except KeyboardInterrupt as e:
            break
        except Exception as e:
            # print e
            resp = response_error()

        conn.sendall(resp)
        conn.close()
<|endoftext|>