<commit_before># -*- coding: utf-8 -*-

import gzip
import json
import os
import shutil
import tempfile

import boto3

from faker import Faker

import moto

from s3backup import main
from s3backup.local_sync_client import LocalSyncClient, traverse
from s3backup.s3_sync_client import S3SyncClient

fake = Faker()

def setup_local_sync_client(target_folder, files):
    for key, timestamp in files.items():
        object_path = os.path.join(target_folder, key)
        with open(object_path, 'w') as fp:
            fp.write(fake.text())
            fp.flush()
            os.utime(object_path, (timestamp, timestamp))

    return LocalSyncClient(target_folder)

class TestPerformSync(object):

    def setup_method(self):
        self.target_folder = tempfile.mkdtemp()

    def teardown_method(self):
        shutil.rmtree(self.target_folder)

    @moto.mock_s3
    def test_perform_sync_empty_local(self):

        client = boto3.client('s3')
        client.create_bucket(Bucket='testbucket')

        index = {
            'foo/bar': {'timestamp': 2000},
            'skeleton/gloves.txt': {'timestamp': 2000},
            'hello.txt': {'timestamp': 2000},
        }
        for key in index.keys():
            client.put_object(Bucket='testbucket', Key='mybackup/{}'.format(key), Body=fake.text())
        client.put_object(
            Bucket='testbucket',
            Key='mybackup/.syncindex.json.gz',
            Body=gzip.compress(json.dumps(index).encode('utf-8')),
        )

        s3_client = S3SyncClient(client, 'testbucket', 'mybackup/')
        local_client = setup_local_sync_client(self.target_folder, files={})

        main.perform_sync(s3_client, local_client)

        object_list = client.list_objects(Bucket='testbucket', Prefix='mybackup/')
        actual_s3_keys = set(
            obj['Key'].replace('mybackup/', '', 1) for obj in object_list['Contents']
        )
        actual_s3_keys.remove('.syncindex.json.gz')

        actual_local_keys = set(traverse(self.target_folder))

        assert actual_local_keys == actual_s3_keys == {
            'foo/bar', 'skeleton/gloves.txt', 'hello.txt'
        }

        for key, metadata in index.items():
            local_timestamp = local_client.get_object_timestamp(key)
            s3_timestamp = s3_client.get_object_timestamp(key)
            target_timestamp = metadata['timestamp']
            assert local_timestamp == s3_timestamp == target_timestamp

    @moto.mock_s3
    def test_perform_sync_empty_s3(self):
        local_files = {'foo': 42323232, 'bar': 34243343}
        local_client = setup_local_sync_client(self.target_folder, files=local_files)

        client = boto3.client('s3')
        client.create_bucket(Bucket='testbucket')
        s3_client = S3SyncClient(client, 'testbucket', 'mybackup/')

        main.perform_sync(s3_client, local_client)

        object_list = client.list_objects(Bucket='testbucket', Prefix='mybackup/')
        actual_s3_keys = set(
            obj['Key'].replace('mybackup/', '', 1) for obj in object_list['Contents']
        )
        actual_s3_keys.remove('.syncindex.json.gz')

        actual_local_keys = set(traverse(self.target_folder))

        assert actual_local_keys == actual_s3_keys == {'foo', 'bar'}

        for key, target_timestamp in local_files.items():
            local_timestamp = local_client.get_object_timestamp(key)
            s3_timestamp = s3_client.get_object_timestamp(key)
            assert local_timestamp == s3_timestamp == target_timestamp
<commit_msg>Cleanup for test_main.py<commit_after># -*- coding: utf-8 -*-

import gzip
import json
import os
import shutil
import tempfile

import boto3

from faker import Faker

import moto

from s3backup import main
from s3backup.local_sync_client import LocalSyncClient, traverse
from s3backup.s3_sync_client import S3SyncClient

fake = Faker()

def setup_local_sync_client(target_folder, objects):
    for key, timestamp in objects.items():
        object_path = os.path.join(target_folder, key)
        with open(object_path, 'w') as fp:
            fp.write(fake.text())
            fp.flush()
            os.utime(object_path, (timestamp, timestamp))

    return LocalSyncClient(target_folder)

def setup_s3_sync_client(client, bucket, prefix, objects, create_index=True):
    client.create_bucket(Bucket='testbucket')
    for key in objects.keys():
        client.put_object(Bucket='testbucket', Key='mybackup/{}'.format(key), Body=fake.text())

    if create_index:
        index = {key: {'timestamp': timestamp} for key, timestamp in objects.items()}
        client.put_object(
            Bucket='testbucket',
            Key='mybackup/.syncindex.json.gz',
            Body=gzip.compress(json.dumps(index).encode('utf-8')),
        )
    return S3SyncClient(client, bucket, prefix)

class TestPerformSync(object):

    def setup_method(self):
        self.target_folder = tempfile.mkdtemp()

    def teardown_method(self):
        shutil.rmtree(self.target_folder)

    @moto.mock_s3
    def test_perform_sync_empty_local(self):
        client = boto3.client('s3')
        objects = {
            'foo/bar': 2020,
            'skeleton/gloves.txt': 5430,
            'hello.txt': 4000,
        }

        s3_client = setup_s3_sync_client(client, 'testbucket', 'mybackup/', objects)
        local_client = setup_local_sync_client(self.target_folder, objects={})

        main.perform_sync(s3_client, local_client)

        object_list = client.list_objects(Bucket='testbucket', Prefix='mybackup/')
        actual_s3_keys = set(
            obj['Key'].replace('mybackup/', '', 1) for obj in object_list['Contents']
        )
        actual_s3_keys.remove('.syncindex.json.gz')

        actual_local_keys = set(traverse(self.target_folder))

        assert actual_local_keys == actual_s3_keys == {
            'foo/bar', 'skeleton/gloves.txt', 'hello.txt'
        }

        for key, target_timestamp in objects.items():
            local_timestamp = local_client.get_object_timestamp(key)
            s3_timestamp = s3_client.get_object_timestamp(key)
            assert local_timestamp == s3_timestamp == target_timestamp

    @moto.mock_s3
    def test_perform_sync_empty_s3(self):
        client = boto3.client('s3')
        objects = {
            'foo': 42323232,
            'bar': 34243343,
        }

        local_client = setup_local_sync_client(self.target_folder, objects=objects)
        s3_client = setup_s3_sync_client(client, 'testbucket', 'mybackup/', {}, False)

        main.perform_sync(s3_client, local_client)

        object_list = client.list_objects(Bucket='testbucket', Prefix='mybackup/')
        actual_s3_keys = set(
            obj['Key'].replace('mybackup/', '', 1) for obj in object_list['Contents']
        )
        actual_s3_keys.remove('.syncindex.json.gz')

        actual_local_keys = set(traverse(self.target_folder))

        assert actual_local_keys == actual_s3_keys == {'foo', 'bar'}

        for key, target_timestamp in objects.items():
            local_timestamp = local_client.get_object_timestamp(key)
            s3_timestamp = s3_client.get_object_timestamp(key)
            assert local_timestamp == s3_timestamp == target_timestamp
<|endoftext|>


<commit_before>import os
from os.path import sep, normpath, join, exists
import ntpath
import copy
from collections import namedtuple
import shutil
from subprocess import Popen, PIPE
import re

from tools.arm_pack_manager import Cache
from tools.targets import TARGET_MAP
from tools.export.exporters import Exporter
from tools.export.cmsis import DeviceCMSIS

cache_d = False

class DeviceUvision(DeviceCMSIS):
    """Uvision Device class, inherits CMSIS Device class

    Encapsulates information necessary for uvision project targets"""
    def __init__(self, target):
        DeviceCMSIS.__init__(self, target)
        dev_format = "$$Device:{0}${1}"
        self.svd = ''
        if self.debug_svd:
            self.svd = dev_format.format(self.dname, self.debug_svd)
        self.reg_file = dev_format.format(self.dname, self.compile_header)
        self.debug_interface = self.uv_debug()
        self.flash_dll = self.generate_flash_dll()

    def uv_debug(self):
        """Return a namedtuple of information about uvision debug settings"""
        UVDebug = namedtuple('UVDebug',['bin_loc','core_flag', 'key'])

        # CortexMXn => pCMX
        cpu = self.core.replace("Cortex-", "C")
        cpu = cpu.replace("+", "")
        cpu = cpu.replace("F", "")
        cpu_flag = "p"+cpu

        # Locations found in Keil_v5/TOOLS.INI
        debuggers = {"st-link": ('STLink\\ST-LINKIII-KEIL_SWO.dll', 'ST-LINKIII-KEIL_SWO'),
                     "j-link":('Segger\\JL2CM3.dll', 'JL2CM3'),
                     "cmsis-dap":('BIN\\CMSIS_AGDI.dll', 'CMSIS_AGDI'),
                     "nulink":('NULink\\Nu_Link.dll','Nu_Link')}
        res = debuggers[self.debug.lower()]
        binary = res[0]
        key = res[1]

        return UVDebug(binary, cpu_flag, key)

    def generate_flash_dll(self):
        '''Flash DLL string from uvision
        S = SW/JTAG Clock ID
        C = CPU index in JTAG chain
        P = Access Port
        For the Options for Target -> Debug tab -> settings -> "Flash" tab in the dialog:
        FD = RAM Start for Flash Functions
        FC = RAM Size for Flash Functions
        FN = Number of Flash types
        FF = Flash File Name (without an extension)
        FS = Start Address of the Flash Device
        FL = Size of the Flash Device
        FP = Full path to the Device algorithm (RTE)

        Necessary to flash some targets. Info gathered from algorithms field of pdsc file.
        '''
        fl_count = 0
        def get_mem_no_x(mem_str):
            mem_reg = "\dx(\w+)"
            m = re.search(mem_reg, mem_str)
            return m.group(1) if m else None

        RAMS = [(get_mem_no_x(info["start"]), get_mem_no_x(info["size"]))
                for mem, info in self.target_info["memory"].items() if "RAM" in mem]
        format_str = "UL2CM3(-S0 -C0 -P0 -FD{ramstart}"+" -FC{ramsize} "+"-FN{num_algos} {extra_flags})"
        ramstart = ''
        #Default according to Keil developer
        ramsize = '1000'
        if len(RAMS)>=1:
            ramstart = RAMS[0][0]
        extra_flags = []
        for name, info in self.target_info["algorithm"].items():
            if not name or not info:
                continue
            if int(info["default"])==0:
                continue
            name_reg = "\w*/([\w_]+)\.flm"
            m = re.search(name_reg, name.lower())
            fl_name = m.group(1) if m else None
            name_flag = "-FF" + str(fl_count) + fl_name

            start, size = get_mem_no_x(info["start"]), get_mem_no_x(info["size"])
            rom_start_flag = "-FS"+str(fl_count)+str(start)
            rom_size_flag = "-FL" + str(fl_count) + str(size)

            if info["ramstart"] is not None and info["ramsize"] is not None:
                ramstart = get_mem_no_x(info["ramstart"])
                ramsize = get_mem_no_x(info["ramsize"])

            path_flag = "-FP" + str(fl_count) + "($$Device:"+self.dname+"$"+name+")"

            extra_flags.extend([name_flag, rom_start_flag, rom_size_flag, path_flag])
            fl_count += 1

        extra = " ".join(extra_flags)
        return format_str.format(ramstart=ramstart,
                                 ramsize=ramsize,
                                 extra_flags=extra, num_algos=fl_count)

class Uvision(Exporter):
    """Keil Uvision class

    This class encapsulates information to be contained in a Uvision
    project file (.uvprojx).
    The needed information can be viewed in uvision.tmpl
    """
    NAME = 'uvision5'
    TOOLCHAIN = 'ARM'
    TARGETS = []
    for target, obj in TARGET_MAP.iteritems():
        if not ("ARM" in obj.supported_toolchains and hasattr(obj, "device_name")):
            continue
        if not DeviceCMSIS.check_supported(target):
            continue
        TARGETS.append(target)
    #File associations within .uvprojx file
    file_types = {'.cpp': 8, '.c': 1, '.s': 2,
                  '.obj': 3, '.o': 3, '.lib': 4,
                  '.ar': 4, '.h': 5, '.hpp': 5, '.sct': 4}

    def uv_file(self, loc):
        """Return a namedtuple of information about project file
        Positional Arguments:
        loc - the file's location

        .uvprojx XML for project file:
        <File>
            <FileType>{{file.type}}</FileType>
            <FileName>{{file.name}}</FileName>
            <FilePath>{{file.loc}}</FilePath>
        </File>
        """
        UVFile = namedtuple('UVFile', ['type','loc','name'])
        _, ext = os.path.splitext(loc)
        type = self.file_types[ext.lower()]
        name = ntpath.basename(normpath(loc))
        return UVFile(type, loc, name)

    def format_flags(self):
        """Format toolchain flags for Uvision"""
        flags = copy.deepcopy(self.flags)
        asm_flag_string = '--cpreproc --cpreproc_opts=-D__ASSERT_MSG,' + \
                          ",".join(flags['asm_flags'])
        # asm flags only, common are not valid within uvision project,
        # they are armcc specific
        flags['asm_flags'] = asm_flag_string
        # cxx flags included, as uvision have them all in one tab
        flags['c_flags'] = list(set(['-D__ASSERT_MSG']
                                        + flags['common_flags']
                                        + flags['c_flags']
                                        + flags['cxx_flags']))
        # not compatible with c99 flag set in the template
        try: flags['c_flags'].remove("--c99")
        except ValueError: pass
        # cpp is not required as it's implicit for cpp files
        try: flags['c_flags'].remove("--cpp")
        except ValueError: pass
        # we want no-vla for only cxx, but it's also applied for C in IDE,
        #  thus we remove it
        try: flags['c_flags'].remove("--no_vla")
        except ValueError: pass
        flags['c_flags'] =" ".join(flags['c_flags'])
        return flags

    def format_src(self, srcs):
        """Make sources into the named tuple for use in the template"""
        grouped = self.group_project_files(srcs)
        for group, files in grouped.items():
            grouped[group] = [self.uv_file(src) for src in files]
        return grouped

    def generate(self):
        """Generate the .uvproj file"""
        cache = Cache(True, False)
        if cache_d:
            cache.cache_descriptors()

        srcs = self.resources.headers + self.resources.s_sources + \
               self.resources.c_sources + self.resources.cpp_sources + \
               self.resources.objects + self.resources.libraries
        ctx = {
            'name': self.project_name,
            'project_files': self.format_src(srcs),
            'linker_script':self.resources.linker_script,
            'include_paths': '; '.join(self.resources.inc_dirs).encode('utf-8'),
            'device': DeviceUvision(self.target),
        }
        # Turn on FPU optimizations if the core has an FPU
        ctx['fpu_setting'] = 1 if 'f' not in ctx['device'].core.lower() \
                                  or 'd' in ctx['device'].core.lower() else 2
        ctx.update(self.format_flags())
        self.gen_file('uvision/uvision.tmpl', ctx, self.project_name+".uvprojx")
        self.gen_file('uvision/uvision_debug.tmpl', ctx, self.project_name + ".uvoptx")

    @staticmethod
    def build(project_name, log_name='build_log.txt', cleanup=True):
        """ Build Uvision project """
        # > UV4 -r -j0 -o [log_name] [project_name].uvprojx
        proj_file = project_name + ".uvprojx"
        cmd = ['UV4', '-r', '-j0', '-o', log_name, proj_file]

        # Build the project
        p = Popen(cmd, stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        ret_code = p.returncode

        # Print the log file to stdout
        with open(log_name, 'r') as f:
            print f.read()

        # Cleanup the exported and built files
        if cleanup:
            os.remove(log_name)
            os.remove(project_name+".uvprojx")
            os.remove(project_name+".uvoptx")
            if exists('.build'):
                shutil.rmtree(".build")

        # Returns 0 upon success, 1 upon a warning, and neither upon an error
        if ret_code != 0 and ret_code != 1:
            # Seems like something went wrong.
            return -1
        else:
            return 0
<commit_msg>[exporters] make sure to clean .build and BUILD when testing uvision<commit_after>import os
from os.path import sep, normpath, join, exists
import ntpath
import copy
from collections import namedtuple
import shutil
from subprocess import Popen, PIPE
import re

from tools.arm_pack_manager import Cache
from tools.targets import TARGET_MAP
from tools.export.exporters import Exporter
from tools.export.cmsis import DeviceCMSIS

cache_d = False

class DeviceUvision(DeviceCMSIS):
    """Uvision Device class, inherits CMSIS Device class

    Encapsulates information necessary for uvision project targets"""
    def __init__(self, target):
        DeviceCMSIS.__init__(self, target)
        dev_format = "$$Device:{0}${1}"
        self.svd = ''
        if self.debug_svd:
            self.svd = dev_format.format(self.dname, self.debug_svd)
        self.reg_file = dev_format.format(self.dname, self.compile_header)
        self.debug_interface = self.uv_debug()
        self.flash_dll = self.generate_flash_dll()

    def uv_debug(self):
        """Return a namedtuple of information about uvision debug settings"""
        UVDebug = namedtuple('UVDebug',['bin_loc','core_flag', 'key'])

        # CortexMXn => pCMX
        cpu = self.core.replace("Cortex-", "C")
        cpu = cpu.replace("+", "")
        cpu = cpu.replace("F", "")
        cpu_flag = "p"+cpu

        # Locations found in Keil_v5/TOOLS.INI
        debuggers = {"st-link": ('STLink\\ST-LINKIII-KEIL_SWO.dll', 'ST-LINKIII-KEIL_SWO'),
                     "j-link":('Segger\\JL2CM3.dll', 'JL2CM3'),
                     "cmsis-dap":('BIN\\CMSIS_AGDI.dll', 'CMSIS_AGDI'),
                     "nulink":('NULink\\Nu_Link.dll','Nu_Link')}
        res = debuggers[self.debug.lower()]
        binary = res[0]
        key = res[1]

        return UVDebug(binary, cpu_flag, key)

    def generate_flash_dll(self):
        '''Flash DLL string from uvision
        S = SW/JTAG Clock ID
        C = CPU index in JTAG chain
        P = Access Port
        For the Options for Target -> Debug tab -> settings -> "Flash" tab in the dialog:
        FD = RAM Start for Flash Functions
        FC = RAM Size for Flash Functions
        FN = Number of Flash types
        FF = Flash File Name (without an extension)
        FS = Start Address of the Flash Device
        FL = Size of the Flash Device
        FP = Full path to the Device algorithm (RTE)

        Necessary to flash some targets. Info gathered from algorithms field of pdsc file.
        '''
        fl_count = 0
        def get_mem_no_x(mem_str):
            mem_reg = "\dx(\w+)"
            m = re.search(mem_reg, mem_str)
            return m.group(1) if m else None

        RAMS = [(get_mem_no_x(info["start"]), get_mem_no_x(info["size"]))
                for mem, info in self.target_info["memory"].items() if "RAM" in mem]
        format_str = "UL2CM3(-S0 -C0 -P0 -FD{ramstart}"+" -FC{ramsize} "+"-FN{num_algos} {extra_flags})"
        ramstart = ''
        #Default according to Keil developer
        ramsize = '1000'
        if len(RAMS)>=1:
            ramstart = RAMS[0][0]
        extra_flags = []
        for name, info in self.target_info["algorithm"].items():
            if not name or not info:
                continue
            if int(info["default"])==0:
                continue
            name_reg = "\w*/([\w_]+)\.flm"
            m = re.search(name_reg, name.lower())
            fl_name = m.group(1) if m else None
            name_flag = "-FF" + str(fl_count) + fl_name

            start, size = get_mem_no_x(info["start"]), get_mem_no_x(info["size"])
            rom_start_flag = "-FS"+str(fl_count)+str(start)
            rom_size_flag = "-FL" + str(fl_count) + str(size)

            if info["ramstart"] is not None and info["ramsize"] is not None:
                ramstart = get_mem_no_x(info["ramstart"])
                ramsize = get_mem_no_x(info["ramsize"])

            path_flag = "-FP" + str(fl_count) + "($$Device:"+self.dname+"$"+name+")"

            extra_flags.extend([name_flag, rom_start_flag, rom_size_flag, path_flag])
            fl_count += 1

        extra = " ".join(extra_flags)
        return format_str.format(ramstart=ramstart,
                                 ramsize=ramsize,
                                 extra_flags=extra, num_algos=fl_count)

class Uvision(Exporter):
    """Keil Uvision class

    This class encapsulates information to be contained in a Uvision
    project file (.uvprojx).
    The needed information can be viewed in uvision.tmpl
    """
    NAME = 'uvision5'
    TOOLCHAIN = 'ARM'
    TARGETS = []
    for target, obj in TARGET_MAP.iteritems():
        if not ("ARM" in obj.supported_toolchains and hasattr(obj, "device_name")):
            continue
        if not DeviceCMSIS.check_supported(target):
            continue
        TARGETS.append(target)
    #File associations within .uvprojx file
    file_types = {'.cpp': 8, '.c': 1, '.s': 2,
                  '.obj': 3, '.o': 3, '.lib': 4,
                  '.ar': 4, '.h': 5, '.hpp': 5, '.sct': 4}

    def uv_file(self, loc):
        """Return a namedtuple of information about project file
        Positional Arguments:
        loc - the file's location

        .uvprojx XML for project file:
        <File>
            <FileType>{{file.type}}</FileType>
            <FileName>{{file.name}}</FileName>
            <FilePath>{{file.loc}}</FilePath>
        </File>
        """
        UVFile = namedtuple('UVFile', ['type','loc','name'])
        _, ext = os.path.splitext(loc)
        type = self.file_types[ext.lower()]
        name = ntpath.basename(normpath(loc))
        return UVFile(type, loc, name)

    def format_flags(self):
        """Format toolchain flags for Uvision"""
        flags = copy.deepcopy(self.flags)
        asm_flag_string = '--cpreproc --cpreproc_opts=-D__ASSERT_MSG,' + \
                          ",".join(flags['asm_flags'])
        # asm flags only, common are not valid within uvision project,
        # they are armcc specific
        flags['asm_flags'] = asm_flag_string
        # cxx flags included, as uvision have them all in one tab
        flags['c_flags'] = list(set(['-D__ASSERT_MSG']
                                        + flags['common_flags']
                                        + flags['c_flags']
                                        + flags['cxx_flags']))
        # not compatible with c99 flag set in the template
        try: flags['c_flags'].remove("--c99")
        except ValueError: pass
        # cpp is not required as it's implicit for cpp files
        try: flags['c_flags'].remove("--cpp")
        except ValueError: pass
        # we want no-vla for only cxx, but it's also applied for C in IDE,
        #  thus we remove it
        try: flags['c_flags'].remove("--no_vla")
        except ValueError: pass
        flags['c_flags'] =" ".join(flags['c_flags'])
        return flags

    def format_src(self, srcs):
        """Make sources into the named tuple for use in the template"""
        grouped = self.group_project_files(srcs)
        for group, files in grouped.items():
            grouped[group] = [self.uv_file(src) for src in files]
        return grouped

    def generate(self):
        """Generate the .uvproj file"""
        cache = Cache(True, False)
        if cache_d:
            cache.cache_descriptors()

        srcs = self.resources.headers + self.resources.s_sources + \
               self.resources.c_sources + self.resources.cpp_sources + \
               self.resources.objects + self.resources.libraries
        ctx = {
            'name': self.project_name,
            'project_files': self.format_src(srcs),
            'linker_script':self.resources.linker_script,
            'include_paths': '; '.join(self.resources.inc_dirs).encode('utf-8'),
            'device': DeviceUvision(self.target),
        }
        # Turn on FPU optimizations if the core has an FPU
        ctx['fpu_setting'] = 1 if 'f' not in ctx['device'].core.lower() \
                                  or 'd' in ctx['device'].core.lower() else 2
        ctx.update(self.format_flags())
        self.gen_file('uvision/uvision.tmpl', ctx, self.project_name+".uvprojx")
        self.gen_file('uvision/uvision_debug.tmpl', ctx, self.project_name + ".uvoptx")

    @staticmethod
    def build(project_name, log_name='build_log.txt', cleanup=True):
        """ Build Uvision project """
        # > UV4 -r -j0 -o [log_name] [project_name].uvprojx
        proj_file = project_name + ".uvprojx"
        cmd = ['UV4', '-r', '-j0', '-o', log_name, proj_file]

        # Build the project
        p = Popen(cmd, stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        ret_code = p.returncode

        # Print the log file to stdout
        with open(log_name, 'r') as f:
            print f.read()

        # Cleanup the exported and built files
        if cleanup:
            os.remove(log_name)
            os.remove(project_name+".uvprojx")
            os.remove(project_name+".uvoptx")
            # legacy .build directory cleaned if exists
            if exists('.build'):
                shutil.rmtree('.build')
            if exists('BUILD'):
                shutil.rmtree('BUILD')

        # Returns 0 upon success, 1 upon a warning, and neither upon an error
        if ret_code != 0 and ret_code != 1:
            # Seems like something went wrong.
            return -1
        else:
            return 0
<|endoftext|>


<commit_before>from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from django import forms
from django.forms import models as modelforms
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.conf import settings

from ella.ellaadmin import widgets
from ella.ellaadmin.options import EllaAdminOptionsMixin
from ella.core.models import Author, Source, Category, Listing, HitCount, Placement

class PlacementForm(modelforms.ModelForm):

    class Meta:
        model = Placement

    def __init__(self, *args, **kwargs):
        initial = []
        if 'initial' in kwargs:
            # FIXME: This is a hotfix only
            # Vypada to na django chybu, kdyz se pouzije distinct(). Do dnesni rev. 8068 oprava neni.
            try:
                initial = [ c.pk for c in Category.objects.distinct().filter(listing__placement=kwargs['initial']['id']) ]
            except IndexError:
                initial = [ c.pk for c in Category.objects.filter(listing__placement=kwargs['initial']['id']) ]

        self.base_fields['listings'] = modelforms.ModelMultipleChoiceField(
                Category.objects.all(), label=_('Category'), cache_choices=True, required=False, initial=initial)
        super(PlacementForm, self).__init__(*args, **kwargs)

class PlacementInlineFormset(generic.GenericInlineFormset):
    def __init__(self, instance=None, data=None, files=None, save_as_new=None):
        self.can_delete = True
        super(PlacementInlineFormset, self).__init__(instance=instance, data=data, files=files)

    def save_existing(self, form, instance, commit=True):
        instance = super(PlacementInlineFormset, self).save_existing(form, instance, commit)
        return self.save_listings(form, instance, commit)

    def save_new(self, form, commit=True):
        instance = super(PlacementInlineFormset, self).save_new(form, commit)
        return self.save_listings(form, instance, commit)

    def save_listings(self, form, instance, commit=True):
        list_cats = form.cleaned_data.pop('listings')

        def save_listings():
            listings = dict([ (l.category, l) for l in Listing.objects.filter(placement=instance.pk) ])

            for c in list_cats:
                if not c in listings:
                    # create listing
                    l = Listing(placement=instance, category=c, publish_from=instance.publish_from)
                    l.save()
                else:
                    del listings[c]
            for l in listings.values():
                l.delete()

        if commit:
            save_listings()
        else:
            save_m2m = form.save_m2m
            def save_all():
                save_m2m()
                save_listings()
            form.save_m2m = save_all
        return instance

    def clean (self):
        # no data - nothing to validate
        if not self.is_valid() or not self.cleaned_data or not self.instance or not self.cleaned_data[0]:
            return

        obj = self.instance
        cat = getattr(obj, 'category', None)
        obj_slug = getattr(obj, 'slug', obj.pk)
        target_ct=ContentType.objects.get_for_model(obj)

        main = None
        for d in self.cleaned_data:
            # empty form
            if not d: break

            if cat and cat == cat and cat:
                main = d

            if d['slug'] and d['slug'] != '':
                slug = d['slug']
            else:
                slug = obj_slug

            # try and find conflicting placement
            qset = Placement.objects.filter(
                category=d['category'],
                slug=slug,
                static=d['static']
)

            if d['static']: # allow placements that do not overlap
                q = Q(publish_to__lt=d['publish_from'])
                if d['publish_to']:
                    q |= Q(publish_from__gt=d['publish_to'])
                qset = qset.exclude(q)

            # check for same date in URL
            if not d['static']:
                qset = qset.filter(
                    publish_from__year=d['publish_from'].year,
                    publish_from__month=d['publish_from'].month,
                    publish_from__day=d['publish_from'].day,
)

            # exclude current object from search
            if d['id']:
                qset = qset.exclude(pk=d['id'])

            if qset:
                plac = qset[0]
                raise forms.ValidationError(
                        _('''There is already a Placement object published in
                        category %(category)s with the same URL referring to %(target)s.
                        Please change the slug or publish date.''') % {
                            'category' : plac.category,
                            'target' : plac.target,
})

            '''
            qset = Placement.objects.filter(
                        target_id=obj.pk,
                        target_ct=target_ct,
)

            if 'id' in d:
                qset = qset.exclude(id=d['id'])

            if qset:
                raise forms.ValidationError('Chyba')
            '''

        if cat and not main:
            raise forms.ValidationError(_('If object has a category, it must have a main placement.'))

        return

class ListingInlineOptions(admin.TabularInline):
    model = Listing
    extra = 2
    fieldsets = ((None, {'fields' : ('category','publish_from', 'priority_from', 'priority_to', 'priority_value', 'remove', 'commercial',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(ListingInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)

class PlacementInlineOptions(generic.GenericTabularInline):
    model = Placement
    extra = 1
    ct_field_name = 'target_ct'
    id_field_name = 'target_id'
    formset = PlacementInlineFormset
    form = PlacementForm
    fieldsets = ((None, {'fields' : ('category', 'publish_from', 'publish_to', 'slug', 'static', 'listings',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(PlacementInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)

class HitCountInlineOptions(admin.TabularInline):
    model = HitCount
    extra = 0

class PlacementOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_display = ('target_admin', 'category', 'publish_from', 'full_url',)
    list_filter = ('publish_from', 'category', 'target_ct',)
    inlines = (ListingInlineOptions,)
    fieldsets = (
        (_('target'), {'fields': ('target_ct', 'target_id', 'slug', 'category',), 'classes': ('wide',)},),
        (_('time'), {'fields': ('publish_from','publish_to', 'static',), 'classes': ('wide',)},),
)

class ListingOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_display = ('target_admin', 'target_ct', 'publish_from', 'category', 'placement_admin', 'target_hitcounts', 'target_url',)
    list_display_links = ()
    list_filter = ('publish_from', 'category__site', 'category', 'placement__target_ct',)
    raw_id_fields = ('placement',)
    date_hierarchy = 'publish_from'

class CategoryOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_filter = ('site',)
    list_display = ('draw_title', 'tree_path', '__unicode__')
    search_fields = ('title', 'slug',)
    ordering = ('site', 'tree_path',)
    prepopulated_fields = {'slug': ('title',)}

class HitCountOptions(admin.ModelAdmin):
    list_display = ('target', 'hits',)
    list_filter = ('placement__target_ct', 'placement__category__site',)

class AuthorOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    prepopulated_fields = {'slug': ('name',)}

class SourceOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_display = ('name', 'url',)

admin.site.register(HitCount, HitCountOptions)
admin.site.register(Category, CategoryOptions)
admin.site.register(Source, SourceOptions)
admin.site.register(Author, AuthorOptions)
admin.site.register(Placement, PlacementOptions)
admin.site.register(Listing, ListingOptions)

<commit_msg>placement validation update - allow different content types with the same date and slug, fixes #1458<commit_after>from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from django import forms
from django.forms import models as modelforms
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.conf import settings

from ella.ellaadmin import widgets
from ella.ellaadmin.options import EllaAdminOptionsMixin
from ella.core.models import Author, Source, Category, Listing, HitCount, Placement

class PlacementForm(modelforms.ModelForm):

    class Meta:
        model = Placement

    def __init__(self, *args, **kwargs):
        initial = []
        if 'initial' in kwargs:
            # FIXME: This is a hotfix only
            # Vypada to na django chybu, kdyz se pouzije distinct(). Do dnesni rev. 8068 oprava neni.
            try:
                initial = [ c.pk for c in Category.objects.distinct().filter(listing__placement=kwargs['initial']['id']) ]
            except IndexError:
                initial = [ c.pk for c in Category.objects.filter(listing__placement=kwargs['initial']['id']) ]

        self.base_fields['listings'] = modelforms.ModelMultipleChoiceField(
                Category.objects.all(), label=_('Category'), cache_choices=True, required=False, initial=initial)
        super(PlacementForm, self).__init__(*args, **kwargs)

class PlacementInlineFormset(generic.GenericInlineFormset):
    def __init__(self, instance=None, data=None, files=None, save_as_new=None):
        self.can_delete = True
        super(PlacementInlineFormset, self).__init__(instance=instance, data=data, files=files)

    def save_existing(self, form, instance, commit=True):
        instance = super(PlacementInlineFormset, self).save_existing(form, instance, commit)
        return self.save_listings(form, instance, commit)

    def save_new(self, form, commit=True):
        instance = super(PlacementInlineFormset, self).save_new(form, commit)
        return self.save_listings(form, instance, commit)

    def save_listings(self, form, instance, commit=True):
        list_cats = form.cleaned_data.pop('listings')

        def save_listings():
            listings = dict([ (l.category, l) for l in Listing.objects.filter(placement=instance.pk) ])

            for c in list_cats:
                if not c in listings:
                    # create listing
                    l = Listing(placement=instance, category=c, publish_from=instance.publish_from)
                    l.save()
                else:
                    del listings[c]
            for l in listings.values():
                l.delete()

        if commit:
            save_listings()
        else:
            save_m2m = form.save_m2m
            def save_all():
                save_m2m()
                save_listings()
            form.save_m2m = save_all
        return instance

    def clean (self):
        # no data - nothing to validate
        if not self.is_valid() or not self.cleaned_data or not self.instance or not self.cleaned_data[0]:
            return

        obj = self.instance
        cat = getattr(obj, 'category', None)
        obj_slug = getattr(obj, 'slug', obj.pk)
        target_ct=ContentType.objects.get_for_model(obj)

        main = None
        for d in self.cleaned_data:
            # empty form
            if not d: break

            if cat and cat == cat and cat:
                main = d

            if d['slug'] and d['slug'] != '':
                slug = d['slug']
            else:
                slug = obj_slug

            # try and find conflicting placement
            qset = Placement.objects.filter(
                category=d['category'],
                slug=slug,
                target_ct=target_ct.pk,
                static=d['static']
)

            if d['static']: # allow placements that do not overlap
                q = Q(publish_to__lt=d['publish_from'])
                if d['publish_to']:
                    q |= Q(publish_from__gt=d['publish_to'])
                qset = qset.exclude(q)

            # check for same date in URL
            if not d['static']:
                qset = qset.filter(
                    publish_from__year=d['publish_from'].year,
                    publish_from__month=d['publish_from'].month,
                    publish_from__day=d['publish_from'].day,
)

            # exclude current object from search
            if d['id']:
                qset = qset.exclude(pk=d['id'])

            if qset:
                plac = qset[0]
                raise forms.ValidationError(
                        _('''There is already a Placement object published in
                        category %(category)s with the same URL referring to %(target)s.
                        Please change the slug or publish date.''') % {
                            'category' : plac.category,
                            'target' : plac.target,
})

            '''
            qset = Placement.objects.filter(
                        target_id=obj.pk,
                        target_ct=target_ct,
)

            if 'id' in d:
                qset = qset.exclude(id=d['id'])

            if qset:
                raise forms.ValidationError('Chyba')
            '''

        if cat and not main:
            raise forms.ValidationError(_('If object has a category, it must have a main placement.'))

        return

class ListingInlineOptions(admin.TabularInline):
    model = Listing
    extra = 2
    fieldsets = ((None, {'fields' : ('category','publish_from', 'priority_from', 'priority_to', 'priority_value', 'remove', 'commercial',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(ListingInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)

class PlacementInlineOptions(generic.GenericTabularInline):
    model = Placement
    extra = 1
    ct_field_name = 'target_ct'
    id_field_name = 'target_id'
    formset = PlacementInlineFormset
    form = PlacementForm
    fieldsets = ((None, {'fields' : ('category', 'publish_from', 'publish_to', 'slug', 'static', 'listings',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(PlacementInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)

class HitCountInlineOptions(admin.TabularInline):
    model = HitCount
    extra = 0

class PlacementOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_display = ('target_admin', 'category', 'publish_from', 'full_url',)
    list_filter = ('publish_from', 'category', 'target_ct',)
    inlines = (ListingInlineOptions,)
    fieldsets = (
        (_('target'), {'fields': ('target_ct', 'target_id', 'slug', 'category',), 'classes': ('wide',)},),
        (_('time'), {'fields': ('publish_from','publish_to', 'static',), 'classes': ('wide',)},),
)

class ListingOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_display = ('target_admin', 'target_ct', 'publish_from', 'category', 'placement_admin', 'target_hitcounts', 'target_url',)
    list_display_links = ()
    list_filter = ('publish_from', 'category__site', 'category', 'placement__target_ct',)
    raw_id_fields = ('placement',)
    date_hierarchy = 'publish_from'

class CategoryOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_filter = ('site',)
    list_display = ('draw_title', 'tree_path', '__unicode__')
    search_fields = ('title', 'slug',)
    ordering = ('site', 'tree_path',)
    prepopulated_fields = {'slug': ('title',)}

class HitCountOptions(admin.ModelAdmin):
    list_display = ('target', 'hits',)
    list_filter = ('placement__target_ct', 'placement__category__site',)

class AuthorOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    prepopulated_fields = {'slug': ('name',)}

class SourceOptions(EllaAdminOptionsMixin, admin.ModelAdmin):
    list_display = ('name', 'url',)

admin.site.register(HitCount, HitCountOptions)
admin.site.register(Category, CategoryOptions)
admin.site.register(Source, SourceOptions)
admin.site.register(Author, AuthorOptions)
admin.site.register(Placement, PlacementOptions)
admin.site.register(Listing, ListingOptions)

<|endoftext|>


<commit_before>import sys
import argparse
import os
import subprocess
import numpy as np
import pm_funcs
from astropy.utils.console import color_print

#PARAMETROS
radio, itera, output, refer, nframes, min_ep, max_err = pm_funcs.get_script()

print 'Iniciando script con...'
print 'linea de comando: %s' % ' '.join(sys.argv)
print 'radio: %d' % radio
print 'itera: %d' % itera
print

#ARGUMENTOS
parser = argparse.ArgumentParser(description='Script PM VVV')
parser.add_argument('<Input List>', help='Lista con inputs (para tlineal)')
parser.add_argument('<Ref Catalog>', help='Catalogo de referencia (usado por PM_1a1)')
parser.add_argument('-c', '--continua', action='store_true', help='Continua desde la ultima iteracion')
parser.add_argument('-p', '--peak', type=int, help='Usa el peak de los PM para calcular nuevas refstars')
parser.add_argument('-r', '--refstars', action='store_true', help='Usa VPD + refstars0, sino solo VPD')

args = parser.parse_args()

#FUNCIONES Y OTROS PARAMETROS
inputs        = vars(args)['<Input List>']#sys.argv[1]
ref_cat       = vars(args)['<Ref Catalog>']#sys.argv[2]
stilts_folder = os.path.dirname(os.path.realpath(__file__))

#continua = False
#if (len(sys.argv) > 3) and (sys.argv[3]=='-C'):
#    continua = True
continua = args.continua

def makedir(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)

#PIPELINE
#output = subprocess.check_output('grep "PDF de Output" %s/tlineal_1a1.py' % stilts_folder, shell=True)
#output = output.split("'")[1]

#refstars = subprocess.check_output('grep "Catalogo con las estrellas de referencia" %s/tlineal_1a1.py' % stilts_folder, shell=True)
#refstars = refstars.split("'")[1]

#nframes = subprocess.check_output('grep "Numero minimo de epocas en que debe estar la estrella" %s/pm_1a1.py' % stilts_folder, shell=True)
#nframes = int(nframes.split(' ')[2])

#min_ep = subprocess.check_output('grep "min_ep =" %s/VPDHmag.py' % stilts_folder, shell=True)
#min_ep = int(min_ep.split(' ')[-1])
refstars = refer
print '\nOutput en tlineal_1a1.py: %s' % output
print 'Archivo de refstars: %s' % refer
print 'nframes en pm_1a1.py: %d' % nframes
print 'min_ep en VPDHmag.py: %d' % min_ep
if continua:
    print 'Continuando desde la ultima iteracion'

nro_files = np.genfromtxt(inputs, unpack=True, dtype='string')
if not nro_files.shape:
    nro_files = np.atleast_1d(nro_files)
nro_files = nro_files.size

if nro_files < nframes:
    print '\nNumero de archivos en el input es menor que nframes!'
    sys.exit()

if nro_files < min_ep:
    print '\nNumero de archivos en el input es menor que min_ep!'
    sys.exit()

if not os.path.isfile('refstars0.gc'):
    print '\nArchivo refstars0.gc no encontrado!'
    sys.exit()

idsr = np.genfromtxt('refstars0.gc', unpack=True, usecols=(0,))

if not continua:
    if os.path.isfile('zelimchisha'):
        print '\nEliminando estrellas de zelimchisha'
        rej_ids = np.genfromtxt('zelimchisha', unpack=True, usecols=(0,))
        id_mask = ~np.in1d(idsr, rej_ids)
        print '\tEncontradas %d estrellas' % (~id_mask).sum()
    else:
        id_mask = np.ones(len(idsr)).astype(bool)

    data = np.genfromtxt('refstars0.gc')
    data = data[id_mask]

    print 'Guardando refstars para primera iteracion'
    np.savetxt(refstars, data, fmt='%f')

    last_idx = 0

    for i in range(1,itera+1):
        if os.path.exists('iter_%d' % i):
            subprocess.call('rm -r iter_%d' % i, shell=True)

if continua:
    import glob
    iter_dirs = np.sort(glob.glob('iter_*'))

    if len(iter_dirs) == 0:
        print 'No hay carpetas iter!'
        sys.exit(1)
    last_idx  = int(iter_dirs[-1].split('_')[-1])

    if not os.path.isfile('iter_%d/PM_final.dat' % last_idx):
        print '\nNo se encontro PM_final.dat en iter_%d!' % last_idx
        sys.exit(1)

    print '\nGenerando nuevo archivo de refstars a partir de la ultima iteracion'
    ids, pmx, pmy, nf, pmxe, pmye = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx), unpack=True, usecols=(0,3,4,6,8,9))
    #Filtro por errores
    pme = np.sqrt(pmxe**2 + pmye**2)
    #Filtro por zelimchisha
    if os.path.isfile('zelimchisha'):
        rej_ids = np.genfromtxt('zelimchisha', unpack=True, usecols=(0,))
        id_mask = ~np.in1d(ids, rej_ids)
    else:
        id_mask = np.ones(len(ids)).astype(bool)
    #Centrar PMs
    if args.peak != None:
        print '\tCalculando peak para centrar refstars'
        from sklearn import mixture
        XG  = np.transpose([pmx, pmy])[(nf >= nframes) * (pme <= max_err) * (id_mask)]
        gmm = mixture.GMM(n_components=args.peak, covariance_type='full').fit(XG)
        idx = np.argmin(np.sum(gmm.means_**2, axis=1))

        x0,y0 = gmm.means_[idx]
        print '\t', x0, y0
    else:
        x0,y0 = 0,0
    #Filtro por radio en PM
    pmr = np.sqrt((pmx-x0)**2 + (pmy-y0)**2)

    if args.refstars:
        pm0 = np.in1d(ids, idsr)
    else:
        pm0 = True

    mask = (pmr <= radio) * (nf >= nframes) * (pme <= max_err) * (id_mask) * (pm0)

    data = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx))
    data = data[mask]

    fmt = '%d %.6f %.6f %.6f %.6f %.3f %d %d %.6f %.6f %.0f %.2f'
    hdr = 'ID RA DEC PM_X PM_Y MAG_K NFRAMES CFRAMES PMXE PMYE NEI NEI_STD'

    np.savetxt(refstars, data, fmt=fmt, header=hdr)

for i in range(itera):
    color_print('\nComenzando iteracion: %d' % (last_idx+i+1), 'lightcyan')

    #Crea carpeta para guardar los outputs
    makedir('iter_%d' % (last_idx+i+1))

    color_print('\tEjecutando tlineal_1a1.py', 'cyan')
    subprocess.call('python -u %s/tlineal_1a1.py %s' % (stilts_folder, inputs), shell=True)

    color_print('\tEjecutando pm_1a1.py', 'cyan')
    subprocess.call('python %s/pm_1a1.py %s' % (stilts_folder, ref_cat), shell=True)

    color_print('\tEjecutando VPDHmag.py', 'cyan')
    subprocess.call('python %s/VPDHmag.py' % stilts_folder, shell=True)

    color_print('\tMoviendo archivos', 'cyan')
    subprocess.call('mv %s*.png iter_%d' % (output, (last_idx+i+1)), shell=True)
    subprocess.call('mv %s iter_%d' % (refstars, (last_idx+i+1)), shell=True)
    makedir('iter_%d/PMs' % (last_idx+i+1))
    subprocess.call('mv PM*.h5 iter_%d' % (last_idx+i+1), shell=True)
    subprocess.call('mv PM_final.dat iter_%d' % (last_idx+i+1), shell=True)
    subprocess.call('mv PMs/* iter_%d/PMs' % (last_idx+i+1), shell=True)
    subprocess.call('mv VPD*.png iter_%d' % (last_idx+i+1), shell=True)

    color_print('\tGenerando nuevo archivo de refstars', 'cyan')
    ids, pmx, pmy, nf, pmex, pmey = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx+i+1), unpack=True, usecols=(0,3,4,6,8,9))
    pme = np.sqrt(pmex**2 + pmey**2)

    if os.path.isfile('zelimchisha'):
        rej_ids = np.genfromtxt('zelimchisha', unpack=True, usecols=(0,))
        id_mask = ~np.in1d(ids, rej_ids)
    else:
        id_mask = np.ones(len(ids)).astype(bool)
    #Centrar PMs
    if args.peak != None:
        print '\tCalculando peak para centrar refstars'
        from sklearn import mixture
        XG  = np.transpose([pmx, pmy])[(nf >= nframes) * (pme <= max_err) * (id_mask)]
        gmm = mixture.GMM(n_components=args.peak, covariance_type='full').fit(XG)
        idx = np.argmin(np.sum(gmm.means_**2, axis=1))

        x0,y0 = gmm.means_[idx]
        print '\t', x0, y0
    else:
        x0,y0 = 0,0

    pmr = np.sqrt((pmx-x0)**2 + (pmy-y0)**2)

    if args.refstars:
        pm0 = np.in1d(ids, idsr)
    else:
        pm0 = True

    mask = (pmr <= radio) * (nf >= nframes) * (pme <= max_err) * (id_mask) * (pm0)

    data = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx+i+1))
    data = data[mask]

    fmt = '%d %.6f %.6f %.6f %.6f %.3f %d %d %.6f %.6f %.0f %.2f'
    hdr = 'ID RA DEC PM_X PM_Y MAG_K NFRAMES CFRAMES PMXE PMYE NEI NEI_STD'

    np.savetxt(refstars, data, fmt=fmt, header=hdr)

subprocess.call('rm -r PMs', shell=True)
subprocess.call('rm %s' % refstars, shell=True)

print '\nDone!'
<commit_msg>Nuevos centros<commit_after>import sys
import argparse
import os
import subprocess
import numpy as np
import pm_funcs
from astropy.utils.console import color_print
from sklearn.neighbors import KernelDensity

#PARAMETROS
radio, itera, output, refer, nframes, min_ep, max_err = pm_funcs.get_script()

print 'Iniciando script con...'
print 'linea de comando: %s' % ' '.join(sys.argv)
print 'radio: %d' % radio
print 'itera: %d' % itera
print

#ARGUMENTOS
parser = argparse.ArgumentParser(description='Script PM VVV')
parser.add_argument('<Input List>', help='Lista con inputs (para tlineal)')
parser.add_argument('<Ref Catalog>', help='Catalogo de referencia (usado por PM_1a1)')
parser.add_argument('-c', '--continua', action='store_true', help='Continua desde la ultima iteracion')
parser.add_argument('-p', '--peak', type=int, help='Usa el peak de los PM para calcular nuevas refstars')
parser.add_argument('-r', '--refstars', action='store_true', help='Usa VPD + refstars0, sino solo VPD')

args = parser.parse_args()

#FUNCIONES Y OTROS PARAMETROS
inputs        = vars(args)['<Input List>']#sys.argv[1]
ref_cat       = vars(args)['<Ref Catalog>']#sys.argv[2]
stilts_folder = os.path.dirname(os.path.realpath(__file__))

#continua = False
#if (len(sys.argv) > 3) and (sys.argv[3]=='-C'):
#    continua = True
continua = args.continua

def makedir(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)

def gaussian(x, amp, mu, sig):
    if (amp < 0) or (sig < 0):
        return np.inf
    return amp * np.exp(-(x-mu)**2 / (2*sig**2))

def two_gaussian(x, amp1, mu1, sig1,
                    amp2, mu2, sig2):
    return (gaussian(x, amp1, mu1, sig1) +
            gaussian(x, amp2, mu2, sig2))

def three_gaussian(x, amp1, mu1, sig1,
                      amp2, mu2, sig2,
                      amp3, mu3, sig3):
    return (gaussian(x, amp1, mu1, sig1) +
    gaussian(x, amp2, mu2, sig2) +
    gaussian(x, amp3, mu3, sig3))

if args.peak == 2:
    gf = two_gaussian
if args.peak == 3:
    gf = three_gaussian

#PIPELINE
#output = subprocess.check_output('grep "PDF de Output" %s/tlineal_1a1.py' % stilts_folder, shell=True)
#output = output.split("'")[1]

#refstars = subprocess.check_output('grep "Catalogo con las estrellas de referencia" %s/tlineal_1a1.py' % stilts_folder, shell=True)
#refstars = refstars.split("'")[1]

#nframes = subprocess.check_output('grep "Numero minimo de epocas en que debe estar la estrella" %s/pm_1a1.py' % stilts_folder, shell=True)
#nframes = int(nframes.split(' ')[2])

#min_ep = subprocess.check_output('grep "min_ep =" %s/VPDHmag.py' % stilts_folder, shell=True)
#min_ep = int(min_ep.split(' ')[-1])
refstars = refer
print '\nOutput en tlineal_1a1.py: %s' % output
print 'Archivo de refstars: %s' % refer
print 'nframes en pm_1a1.py: %d' % nframes
print 'min_ep en VPDHmag.py: %d' % min_ep
if continua:
    print 'Continuando desde la ultima iteracion'

nro_files = np.genfromtxt(inputs, unpack=True, dtype='string')
if not nro_files.shape:
    nro_files = np.atleast_1d(nro_files)
nro_files = nro_files.size

if nro_files < nframes:
    print '\nNumero de archivos en el input es menor que nframes!'
    sys.exit()

if nro_files < min_ep:
    print '\nNumero de archivos en el input es menor que min_ep!'
    sys.exit()

if not os.path.isfile('refstars0.gc'):
    print '\nArchivo refstars0.gc no encontrado!'
    sys.exit()

idsr = np.genfromtxt('refstars0.gc', unpack=True, usecols=(0,))

if not continua:
    if os.path.isfile('zelimchisha'):
        print '\nEliminando estrellas de zelimchisha'
        rej_ids = np.genfromtxt('zelimchisha', unpack=True, usecols=(0,))
        id_mask = ~np.in1d(idsr, rej_ids)
        print '\tEncontradas %d estrellas' % (~id_mask).sum()
    else:
        id_mask = np.ones(len(idsr)).astype(bool)

    data = np.genfromtxt('refstars0.gc')
    data = data[id_mask]

    print 'Guardando refstars para primera iteracion'
    np.savetxt(refstars, data, fmt='%f')

    last_idx = 0

    for i in range(1,itera+1):
        if os.path.exists('iter_%d' % i):
            subprocess.call('rm -r iter_%d' % i, shell=True)

if continua:
    import glob
    iter_dirs = np.sort(glob.glob('iter_*'))

    if len(iter_dirs) == 0:
        print 'No hay carpetas iter!'
        sys.exit(1)
    last_idx  = int(iter_dirs[-1].split('_')[-1])

    if not os.path.isfile('iter_%d/PM_final.dat' % last_idx):
        print '\nNo se encontro PM_final.dat en iter_%d!' % last_idx
        sys.exit(1)

    print '\nGenerando nuevo archivo de refstars a partir de la ultima iteracion'
    ids, pmx, pmy, nf, pmxe, pmye = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx), unpack=True, usecols=(0,3,4,6,8,9))
    #Filtro por errores
    pme = np.sqrt(pmxe**2 + pmye**2)
    #Filtro por zelimchisha
    if os.path.isfile('zelimchisha'):
        rej_ids = np.genfromtxt('zelimchisha', unpack=True, usecols=(0,))
        id_mask = ~np.in1d(ids, rej_ids)
    else:
        id_mask = np.ones(len(ids)).astype(bool)
    #Centrar PMs
    if args.peak != None:
        print '\tCalculando peak para centrar refstars'
        from sklearn import mixture
        XG    = np.transpose([pmx, pmy])[(nf >= nframes) * (pme <= max_err) * (id_mask) * (pm1)]
        gmm   = mixture.GMM(n_components=args.peak, covariance_type='full').fit(XG)
        xg,yg = np.transpose(g.means_)

        #Centros x
        kde = KernelDensity(kernel='gaussian').fit(XG[:,0][:, np.newaxis])
        xx  = np.arange(-15, 15, 0.05)
        yy  = np.exp(kde.score_samples(xx[:,np.newaxis]))
        a0  = np.exp(kde.score_samples(xg[:,np.newaxis]))
        if args.peak == 3:
            p0  = [a0[0]/2.0, xg[0], 3, a0[1]/2.0, xg[1], 3, a0[2]/2.0, xg[2], 3]
        elif args.peak == 2:
            p0  = [a0[0]/2.0, xg[0], 3, a0[1]/2.0, xg[1], 3]

        popt, pcov = curve_fit(gf, xx, yy, p0=p0, maxfev=10000)
        x0g = popt[1::3]
        x0e = np.sqrt(np.diag(pcov)[1::3])

        #Centros y
        kde = KernelDensity(kernel='gaussian').fit(XG[:,1][:, np.newaxis])
        yy  = np.exp(kde.score_samples(xx[:,np.newaxis]))
        a0  = np.exp(kde.score_samples(yg[:,np.newaxis]))
        if args.peak == 3:
            p0  = [a0[0]/2.0, yg[0], 3, a0[1]/2.0, yg[1], 3, a0[2]/2.0, yg[2], 3]
        elif args.peak == 2:
            p0  = [a0[0]/2.0, yg[0], 3, a0[1]/2.0, yg[1], 3]

        popt, pcov = curve_fit(gf, xx, yy, p0=p0, maxfev=10000)
        y0g = popt[1::3]
        y0e = np.sqrt(np.diag(pcov)[1::3])

        cen = np.transpose([x0g, y0g])
        idx = np.argmin(np.sum(cen**2, axis=1))

        if np.sum(cen[idx]**2) > 3:
            x0,y0 = gmm.means_[idx]
            print '\tCentros muy grandes, volviendo a los guess'
        else:
            x0,y0 = cen[idx]
        print '\t', x0, y0
    else:
        x0,y0 = 0,0
    #Filtro por radio en PM
    pmr = np.sqrt((pmx-x0)**2 + (pmy-y0)**2)

    if args.refstars:
        pm0 = np.in1d(ids, idsr)
    else:
        pm0 = True

    mask = (pmr <= radio) * (nf >= nframes) * (pme <= max_err) * (id_mask) * (pm0)

    data = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx))
    data = data[mask]

    fmt = '%d %.6f %.6f %.6f %.6f %.3f %d %d %.6f %.6f %.0f %.2f'
    hdr = 'ID RA DEC PM_X PM_Y MAG_K NFRAMES CFRAMES PMXE PMYE NEI NEI_STD'

    np.savetxt(refstars, data, fmt=fmt, header=hdr)

for i in range(itera):
    color_print('\nComenzando iteracion: %d' % (last_idx+i+1), 'lightcyan')

    #Crea carpeta para guardar los outputs
    makedir('iter_%d' % (last_idx+i+1))

    color_print('\tEjecutando tlineal_1a1.py', 'cyan')
    subprocess.call('python -u %s/tlineal_1a1.py %s' % (stilts_folder, inputs), shell=True)

    color_print('\tEjecutando pm_1a1.py', 'cyan')
    subprocess.call('python %s/pm_1a1.py %s' % (stilts_folder, ref_cat), shell=True)

    color_print('\tEjecutando VPDHmag.py', 'cyan')
    subprocess.call('python %s/VPDHmag.py' % stilts_folder, shell=True)

    color_print('\tMoviendo archivos', 'cyan')
    subprocess.call('mv %s*.png iter_%d' % (output, (last_idx+i+1)), shell=True)
    subprocess.call('mv %s iter_%d' % (refstars, (last_idx+i+1)), shell=True)
    makedir('iter_%d/PMs' % (last_idx+i+1))
    subprocess.call('mv PM*.h5 iter_%d' % (last_idx+i+1), shell=True)
    subprocess.call('mv PM_final.dat iter_%d' % (last_idx+i+1), shell=True)
    subprocess.call('mv PMs/* iter_%d/PMs' % (last_idx+i+1), shell=True)
    subprocess.call('mv VPD*.png iter_%d' % (last_idx+i+1), shell=True)

    color_print('\tGenerando nuevo archivo de refstars', 'cyan')
    ids, pmx, pmy, nf, pmex, pmey = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx+i+1), unpack=True, usecols=(0,3,4,6,8,9))
    pme = np.sqrt(pmex**2 + pmey**2)
    pm1 = (pmx**2 + pmy**2)**0.5 < 30

    if os.path.isfile('zelimchisha'):
        rej_ids = np.genfromtxt('zelimchisha', unpack=True, usecols=(0,))
        id_mask = ~np.in1d(ids, rej_ids)
    else:
        id_mask = np.ones(len(ids)).astype(bool)
    #Centrar PMs
    if args.peak != None:
        print '\tCalculando peak para centrar refstars'
        from sklearn import mixture
        XG    = np.transpose([pmx, pmy])[(nf >= nframes) * (pme <= max_err) * (id_mask) * (pm1)]
        gmm   = mixture.GMM(n_components=args.peak, covariance_type='full').fit(XG)
        xg,yg = np.transpose(g.means_)

        #Centros x
        kde = KernelDensity(kernel='gaussian').fit(XG[:,0][:, np.newaxis])
        xx  = np.arange(-15, 15, 0.05)
        yy  = np.exp(kde.score_samples(xx[:,np.newaxis]))
        a0  = np.exp(kde.score_samples(xg[:,np.newaxis]))
        p0  = [a0[0]/2.0, xg[0], 3, a0[1]/2.0, xg[1], 3]

        popt, pcov = curve_fit(gf, xx, yy, p0=p0, maxfev=10000)
        x0g = popt[1::3]
        x0e = np.sqrt(np.diag(pcov)[1::3])

        #Centros y
        kde = KernelDensity(kernel='gaussian').fit(XG[:,1][:, np.newaxis])
        yy  = np.exp(kde.score_samples(xx[:,np.newaxis]))
        a0  = np.exp(kde.score_samples(yg[:,np.newaxis]))
        p0  = [a0[0]/2.0, yg[0], 3, a0[1]/2.0, yg[1], 3]

        popt, pcov = curve_fit(gf, xx, yy, p0=p0, maxfev=10000)
        y0g = popt[1::3]
        y0e = np.sqrt(np.diag(pcov)[1::3])

        cen = np.transpose([x0g, y0g])
        idx = np.argmin(np.sum(cen**2, axis=1))

        if np.sum(cen[idx]**2) > 3:
            x0,y0 = gmm.means_[idx]
            print '\tCentros muy grandes, volviendo a los guess'
        else:
            x0,y0 = cen[idx]
        print '\t', x0, y0
    else:
        x0,y0 = 0,0

    pmr = np.sqrt((pmx-x0)**2 + (pmy-y0)**2)

    if args.refstars:
        pm0 = np.in1d(ids, idsr)
    else:
        pm0 = True

    mask = (pmr <= radio) * (nf >= nframes) * (pme <= max_err) * (id_mask) * (pm0)

    data = np.genfromtxt('iter_%d/PM_final.dat' % (last_idx+i+1))
    data = data[mask]

    fmt = '%d %.6f %.6f %.6f %.6f %.3f %d %d %.6f %.6f %.0f %.2f'
    hdr = 'ID RA DEC PM_X PM_Y MAG_K NFRAMES CFRAMES PMXE PMYE NEI NEI_STD'

    np.savetxt(refstars, data, fmt=fmt, header=hdr)

subprocess.call('rm -r PMs', shell=True)
subprocess.call('rm %s' % refstars, shell=True)

print '\nDone!'
<|endoftext|>


<commit_before>from mimetypes import guess_type

from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Atom1Feed
from django.http import Http404
from django.template import TemplateDoesNotExist, RequestContext, NodeList

from ella.core.models import Listing, Category
from ella.core.conf import core_settings
from ella.core.managers import ListingHandler
from ella.core.box import Box
from ella.photos.models import Format, FormatedPhoto

class RSSTopCategoryListings(Feed):
    def __init__(self, *args, **kwargs):
        super(RSSTopCategoryListings, self).__init__(*args, **kwargs)
        if core_settings.RSS_ENCLOSURE_PHOTO_FORMAT:
            self.format = Format.objects.get_for_name(core_settings.RSS_ENCLOSURE_PHOTO_FORMAT)
        else:
            self.format = None

    def get_object(self, request, category=''):
        bits = category.split('/')
        try:
            cat = Category.objects.get_by_tree_path(u'/'.join(bits))
        except Category.DoesNotExist:
            raise Http404()

        self.box_context = RequestContext(request)

        return cat

    def items(self, obj):
        kwa = {}
        kwa['category'] = obj

        # TODO: In ella based application children attr can be NONE, IMMEDIATE and ALL
        if kwa['category'].tree_parent != None:
            kwa['children'] = ListingHandler.ALL

        qset = Listing.objects.get_queryset_wrapper(**kwa)
        return qset.get_listings(count=core_settings.RSS_NUM_IN_FEED)

    # Feed metadata
    ###########################################################################
    def title(self, obj):
        return obj.app_data.get('syndication', {}).get('title', obj.title)

    def link(self, obj):
        return obj.get_absolute_url()

    def description(self, obj):
        return obj.app_data.get('syndication', {}).get('title', obj.description)

    # Item metadata
    ###########################################################################
    def item_pubdate(self, item):
        return item.publish_from

    def item_title(self, item):
        return item.publishable.title

    def item_link(self, item):
        return item.get_absolute_url()

    def item_description(self, item):
        if not core_settings.RSS_DESCRIPTION_BOX_TYPE:
            return item.publishable.description

        p = item.publishable
        box = p.box_class(p, core_settings.RSS_DESCRIPTION_BOX_TYPE, NodeList())
        try:
            return box.render(self.box_context)
        except TemplateDoesNotExist:
            return item.publishable.description

    # Enclosure - Photo
    ###########################################################################
    def item_enclosure_url(self, item):
        if not hasattr(item, '__enclosure_url'):
            if item.publishable.photo_id:
                if self.format:
                    item.__enclosure_url = FormatedPhoto.objects.get_photo_in_format(item.publishable.photo_id, self.format)['url']
                else:
                    item.__enclosure_url = item.publishable.photo.image.url
            else:
                item.__enclosure_url = None

        return item.__enclosure_url

    def item_enclosure_mime_type(self, item):
        im = self.item_enclosure_url(item)
        if im:
            return guess_type(im)[0]

    def item_enclosure_size(self, item):
        # make sure get_photo_in_format was called
        if self.item_enclosure_url(item):
            if self.format:
                return FormatedPhoto.objects.get(photo=item.publishable.photo_id, format=self.format).image.size
            else:
                return item.publishable.photo.image.size

class AtomTopCategoryListings(RSSTopCategoryListings):
    feed_type = Atom1Feed
    subtitle = RSSTopCategoryListings.description

<commit_msg>copy-paste<commit_after>from mimetypes import guess_type

from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Atom1Feed
from django.http import Http404
from django.template import TemplateDoesNotExist, RequestContext, NodeList

from ella.core.models import Listing, Category
from ella.core.conf import core_settings
from ella.core.managers import ListingHandler
from ella.core.box import Box
from ella.photos.models import Format, FormatedPhoto

class RSSTopCategoryListings(Feed):
    def __init__(self, *args, **kwargs):
        super(RSSTopCategoryListings, self).__init__(*args, **kwargs)
        if core_settings.RSS_ENCLOSURE_PHOTO_FORMAT:
            self.format = Format.objects.get_for_name(core_settings.RSS_ENCLOSURE_PHOTO_FORMAT)
        else:
            self.format = None

    def get_object(self, request, category=''):
        bits = category.split('/')
        try:
            cat = Category.objects.get_by_tree_path(u'/'.join(bits))
        except Category.DoesNotExist:
            raise Http404()

        self.box_context = RequestContext(request)

        return cat

    def items(self, obj):
        kwa = {}
        kwa['category'] = obj

        # TODO: In ella based application children attr can be NONE, IMMEDIATE and ALL
        if kwa['category'].tree_parent != None:
            kwa['children'] = ListingHandler.ALL

        qset = Listing.objects.get_queryset_wrapper(**kwa)
        return qset.get_listings(count=core_settings.RSS_NUM_IN_FEED)

    # Feed metadata
    ###########################################################################
    def title(self, obj):
        return obj.app_data.get('syndication', {}).get('title', obj.title)

    def link(self, obj):
        return obj.get_absolute_url()

    def description(self, obj):
        return obj.app_data.get('syndication', {}).get('description', obj.description)

    # Item metadata
    ###########################################################################
    def item_pubdate(self, item):
        return item.publish_from

    def item_title(self, item):
        return item.publishable.title

    def item_link(self, item):
        return item.get_absolute_url()

    def item_description(self, item):
        if not core_settings.RSS_DESCRIPTION_BOX_TYPE:
            return item.publishable.description

        p = item.publishable
        box = p.box_class(p, core_settings.RSS_DESCRIPTION_BOX_TYPE, NodeList())
        try:
            return box.render(self.box_context)
        except TemplateDoesNotExist:
            return item.publishable.description

    # Enclosure - Photo
    ###########################################################################
    def item_enclosure_url(self, item):
        if not hasattr(item, '__enclosure_url'):
            if item.publishable.photo_id:
                if self.format:
                    item.__enclosure_url = FormatedPhoto.objects.get_photo_in_format(item.publishable.photo_id, self.format)['url']
                else:
                    item.__enclosure_url = item.publishable.photo.image.url
            else:
                item.__enclosure_url = None

        return item.__enclosure_url

    def item_enclosure_mime_type(self, item):
        im = self.item_enclosure_url(item)
        if im:
            return guess_type(im)[0]

    def item_enclosure_size(self, item):
        # make sure get_photo_in_format was called
        if self.item_enclosure_url(item):
            if self.format:
                return FormatedPhoto.objects.get(photo=item.publishable.photo_id, format=self.format).image.size
            else:
                return item.publishable.photo.image.size

class AtomTopCategoryListings(RSSTopCategoryListings):
    feed_type = Atom1Feed
    subtitle = RSSTopCategoryListings.description

<|endoftext|>


<commit_before>import datetime

from django.contrib.contenttypes.models import ContentType
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.core.paginator import ObjectPaginator
from django.conf import settings

from ella.core.models import Listing, Category, HitCount
from ella.core.cache import *
from ella.core.custom_urls import dispatcher

CONTENT_TYPE_MAPPING = {}
def get_content_type(ct_name):
    """
    A helper function that returns ContentType object based on its slugified verbose_name_plural.

    Results of this function is cached to improve performance.

    Params:
        ct_name - slugified verbose_name_plural of the target model.

    Raises:
        Http404 if no matching ContentType is found
    """
    try:
        ct = CONTENT_TYPE_MAPPING[ct_name]
    except KeyError:
        from django.db import models
        from django.template.defaultfilters import slugify
        for model in models.get_models():
            if ct_name == slugify(model._meta.verbose_name_plural):
                ct = ContentType.objects.get_for_model(model)
                CONTENT_TYPE_MAPPING[ct_name] = ct
                break
        else:
            raise Http404
    return ct

def object_detail(request, category, year, month, day, content_type, slug, url_remainder=None):
    """
    Detail view that displays a single object based on it's main listing.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path (empty if home category)
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
        content_type - slugified verbose_name_plural of the target model
        slug - slug of the object itself

    Raises:
        Http404 if object or listing doesn't exist or dates doesn't match
    """
    ct = get_content_type(content_type)

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)

    obj = get_cached_object_or_404(ct, slug=slug)
    try:
        date = datetime.date(int(year), int(month), int(day))
        listing = get_cached_object_or_404(Listing, target_ct=ct, target_id=obj.id, category=cat)
        if listing.publish_from.date() != date:
            raise Http404

    except ValueError:
        # incorrect date
        raise Http404

    context = {
            'listing' : listing,
            'object' : obj,
            'category' : cat,
            'content_type_name' : content_type,
            'content_type' : ct,
}

    # check for custom actions
    if url_remainder:
        bits = url_remainder.split('/')
        return dispatcher.call_view(request, bits, context)
    elif dispatcher.has_custom_detail(obj):
        # increment hit counter
        HitCount.objects.hit(obj)
        return dispatcher.call_custom_detail(request, context)

    # increment hit counter
    HitCount.objects.hit(obj)

    return render_to_response(
            (
                # NEW
                'page/category/%s/content_type/%s.%s/%s/object.html' % (cat.path, ct.app_label, ct.model, slug),
                'page/category/%s/content_type/%s.%s/object.html' % (cat.path, ct.app_label, ct.model),
                'page/category/%s/object.html' % (cat.path),
                'page/content_type/%s.%s/object.html' % (ct.app_label, ct.model),
                'page/object.html',

                # OLD
                'category/%s/%s.%s/%s_detail.html' % (cat.path, ct.app_label, ct.model, slug),
                'category/%s/%s.%s/base_detail.html' % (cat.path, ct.app_label, ct.model),
                'category/%s/base_detail.html' % (cat.path),
                'core/object_detail.html',
),
            context,
            context_instance=RequestContext(request)
)

def list_content_type(request, category=None, year=None, month=None, day=None, content_type=None, paginate_by=20):
    """
    List object's listings according to the parameters.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path, optional - defaults to all categories
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
                           All of these parameters are optional, the list will be filtered by the non-empty ones
        content_type - slugified verbose_name_plural of the target model, if omitted all content_types are listed

    Raises:
        Http404 - if the specified category or content_type does not exist or if the given date is malformed.
    """
    kwa = {}
    dates_kwa = {}
    if year:
        current_date = datetime.datetime(int(year), 1, 1)
        dates_kwa['publish_from__year'] = int(year)
        date_kind = 'month'

    if month:
        try:
            current_date = datetime.datetime(int(year), int(month), 1)
        except ValueError:
            raise Http404
        dates_kwa['publish_from__month'] = int(month)
        date_kind = 'day'

    if day:
        try:
            current_date = datetime.datetime(int(year), int(month), int(day))
        except ValueError:
            raise Http404
        dates_kwa['publish_from__day'] = int(day)
        date_kind = 'detail'

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    kwa['category'] = cat

    if content_type:
        ct = get_content_type(content_type)
        kwa['content_types'] = [ ct ]
    else:
        ct = False

    # pagination
    if 'p' in request.GET and request.GET['p'].isdigit():
        page = int(request.GET['p'])
    else:
        page = 1

    base_qset = Listing.objects.get_queryset(**kwa)
    kwa.update(dates_kwa)
    qset = Listing.objects.get_queryset(**kwa)
    paginator = ObjectPaginator(qset.filter(**dates_kwa), paginate_by)

    if page > paginator.pages:
        raise Http404

    kwa['count'] = paginate_by
    kwa['offset'] = (page - 1) * paginate_by + 1
    listings = Listing.objects.get_listing(**kwa)

    template_list = []
    if ct:
        # NEW
        template_list.append('page/category/%s/content_type/%s.%s/listing.html' % (cat.path, ct.app_label, ct.model))
        # OLD
        template_list.append('category/%s/%s.%s/list.html' % (cat.path, ct.app_label, ct.model))
    # NEW
    template_list.append('page/category/%s/listing.html' % (cat.path))

    if ct:
        # NEW
        template_list.append('page/content_type/%s.%s/listing.html' % (ct.app_label, ct.model))

    # NEW
    template_list.append('page/listing.html')

    # OLD
    template_list.append('category/%s/base_list.html' % (cat.path))
    template_list.append('core/base_list.html')

    return render_to_response(template_list, {
            'is_paginated': paginator.pages > 1,
            'results_per_page': paginate_by,
            'has_next': paginator.has_next_page(page - 1),
            'has_previous': paginator.has_previous_page(page - 1),
            'page': page,
            'next': page + 1,
            'previous': page - 1,
            'last_on_page': paginator.last_on_page(page - 1),
            'first_on_page': paginator.first_on_page(page - 1),
            'pages': paginator.pages,
            'hits' : paginator.hits,

            'year_list' : [ (d, d.strftime(DATE_REPR["year"]), d.strftime(YEAR_URLS[date_kind])) for d in base_qset.dates('publish_from', "year", order='DESC') ],
            'date_list' : date_kind != 'detail' and [ (d, d.strftime(DATE_REPR[date_kind]), d.strftime(DATE_URLS[date_kind])) for d in qset.dates('publish_from', date_kind, order='DESC') ] or None,
            'current_date' : current_date,
            'current_date_text' : current_date.strftime(CURRENT_DATE_REPR[date_kind]),
            'date_kind' : date_kind,

            'content_type' : content_type,
            'listings' : listings,
            'category' : cat,
}, context_instance=RequestContext(request))

DATE_URLS = {'year' : '../%Y/', 'month' : '../../%Y/%m/', 'day' : '../../../%Y/%m/%d/',}
YEAR_URLS = {'year' : '../%Y/', 'month' : '../../%Y/', 'day' : '../../../%Y/', 'detail' : '../../../%Y/'}
DATE_REPR = {'year' : '%Y', 'month' : '%m/%Y', 'day' : '%d/%m/%Y', 'detail' : '%d/%m/%Y',}
CURRENT_DATE_REPR = {'year' : '', 'month' : '%Y', 'day' : '%m/%Y', 'detail' : '%d/%m/%Y',}

def home(request):
    """
    Homepage of the actual site.

    Params:
        request - HttpRequest from Django

    Raise:
        Http404 if there is no base category
    """
    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    return render_to_response(
            (
                # NEW
                'page/category/%s/category.html' % (cat.slug),
                'page/category.html',

                # OLD
                'category/%s/detail.html' % (cat.slug),
                'core/category_detail.html',
),
            {
                'category' : cat,
},
            context_instance=RequestContext(request)
)

def category_detail(request, category):
    """
    Homepage of a given category.

    Params:
        request - HttpRequest from Django
        category - category's ``tree_path``

    Raise:
        Http404 if the category doesn't exist
    """
    cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    return render_to_response(
            (
                # NEW
                'page/category/%s/category.html' % (cat.slug),
                'page/category.html',

                # OLD
                'category/%s/detail.html' % (cat.slug),
                'core/category_detail.html',
),
            {
                'category' : cat
},
            context_instance=RequestContext(request)
)
def export_test(*args, **kwargs):
    return []

@cache_this(method_key_getter, export_test, timeout=60*60)
def export(request, count, models=None):
    if models is None:
        from ella.articles.models import Article
        models = [ Article, ]

    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    listing = Listing.objects.get_listing(count=count, category=cat, mods=models)
    return render_to_response(
            ('page/export_banner.html', 'export_banner.html',),
            {'category' : cat, 'listing' : listing},
            context_instance=RequestContext(request)
)

def page_not_found(request):
    response = render_to_response('page/404.html', {}, context_instance=RequestContext(request))
    response.status_code = 404
    return response

def handle_error(request):
    response = render_to_response('page/500.html', {}, context_instance=RequestContext(request))
    response.status_code = 500
    return response
<commit_msg>Fixed wrong cache generation for export banner - it didn't include SITE_ID.<commit_after>import datetime

from django.contrib.contenttypes.models import ContentType
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.core.paginator import ObjectPaginator
from django.conf import settings

from ella.core.models import Listing, Category, HitCount
from ella.core.cache import *
from ella.core.custom_urls import dispatcher

CONTENT_TYPE_MAPPING = {}
def get_content_type(ct_name):
    """
    A helper function that returns ContentType object based on its slugified verbose_name_plural.

    Results of this function is cached to improve performance.

    Params:
        ct_name - slugified verbose_name_plural of the target model.

    Raises:
        Http404 if no matching ContentType is found
    """
    try:
        ct = CONTENT_TYPE_MAPPING[ct_name]
    except KeyError:
        from django.db import models
        from django.template.defaultfilters import slugify
        for model in models.get_models():
            if ct_name == slugify(model._meta.verbose_name_plural):
                ct = ContentType.objects.get_for_model(model)
                CONTENT_TYPE_MAPPING[ct_name] = ct
                break
        else:
            raise Http404
    return ct

def object_detail(request, category, year, month, day, content_type, slug, url_remainder=None):
    """
    Detail view that displays a single object based on it's main listing.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path (empty if home category)
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
        content_type - slugified verbose_name_plural of the target model
        slug - slug of the object itself

    Raises:
        Http404 if object or listing doesn't exist or dates doesn't match
    """
    ct = get_content_type(content_type)

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)

    obj = get_cached_object_or_404(ct, slug=slug)
    try:
        date = datetime.date(int(year), int(month), int(day))
        listing = get_cached_object_or_404(Listing, target_ct=ct, target_id=obj.id, category=cat)
        if listing.publish_from.date() != date:
            raise Http404

    except ValueError:
        # incorrect date
        raise Http404

    context = {
            'listing' : listing,
            'object' : obj,
            'category' : cat,
            'content_type_name' : content_type,
            'content_type' : ct,
}

    # check for custom actions
    if url_remainder:
        bits = url_remainder.split('/')
        return dispatcher.call_view(request, bits, context)
    elif dispatcher.has_custom_detail(obj):
        # increment hit counter
        HitCount.objects.hit(obj)
        return dispatcher.call_custom_detail(request, context)

    # increment hit counter
    HitCount.objects.hit(obj)

    return render_to_response(
            (
                # NEW
                'page/category/%s/content_type/%s.%s/%s/object.html' % (cat.path, ct.app_label, ct.model, slug),
                'page/category/%s/content_type/%s.%s/object.html' % (cat.path, ct.app_label, ct.model),
                'page/category/%s/object.html' % (cat.path),
                'page/content_type/%s.%s/object.html' % (ct.app_label, ct.model),
                'page/object.html',

                # OLD
                'category/%s/%s.%s/%s_detail.html' % (cat.path, ct.app_label, ct.model, slug),
                'category/%s/%s.%s/base_detail.html' % (cat.path, ct.app_label, ct.model),
                'category/%s/base_detail.html' % (cat.path),
                'core/object_detail.html',
),
            context,
            context_instance=RequestContext(request)
)

def list_content_type(request, category=None, year=None, month=None, day=None, content_type=None, paginate_by=20):
    """
    List object's listings according to the parameters.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path, optional - defaults to all categories
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
                           All of these parameters are optional, the list will be filtered by the non-empty ones
        content_type - slugified verbose_name_plural of the target model, if omitted all content_types are listed

    Raises:
        Http404 - if the specified category or content_type does not exist or if the given date is malformed.
    """
    kwa = {}
    dates_kwa = {}
    if year:
        current_date = datetime.datetime(int(year), 1, 1)
        dates_kwa['publish_from__year'] = int(year)
        date_kind = 'month'

    if month:
        try:
            current_date = datetime.datetime(int(year), int(month), 1)
        except ValueError:
            raise Http404
        dates_kwa['publish_from__month'] = int(month)
        date_kind = 'day'

    if day:
        try:
            current_date = datetime.datetime(int(year), int(month), int(day))
        except ValueError:
            raise Http404
        dates_kwa['publish_from__day'] = int(day)
        date_kind = 'detail'

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    kwa['category'] = cat

    if content_type:
        ct = get_content_type(content_type)
        kwa['content_types'] = [ ct ]
    else:
        ct = False

    # pagination
    if 'p' in request.GET and request.GET['p'].isdigit():
        page = int(request.GET['p'])
    else:
        page = 1

    base_qset = Listing.objects.get_queryset(**kwa)
    kwa.update(dates_kwa)
    qset = Listing.objects.get_queryset(**kwa)
    paginator = ObjectPaginator(qset.filter(**dates_kwa), paginate_by)

    if page > paginator.pages:
        raise Http404

    kwa['count'] = paginate_by
    kwa['offset'] = (page - 1) * paginate_by + 1
    listings = Listing.objects.get_listing(**kwa)

    template_list = []
    if ct:
        # NEW
        template_list.append('page/category/%s/content_type/%s.%s/listing.html' % (cat.path, ct.app_label, ct.model))
        # OLD
        template_list.append('category/%s/%s.%s/list.html' % (cat.path, ct.app_label, ct.model))
    # NEW
    template_list.append('page/category/%s/listing.html' % (cat.path))

    if ct:
        # NEW
        template_list.append('page/content_type/%s.%s/listing.html' % (ct.app_label, ct.model))

    # NEW
    template_list.append('page/listing.html')

    # OLD
    template_list.append('category/%s/base_list.html' % (cat.path))
    template_list.append('core/base_list.html')

    return render_to_response(template_list, {
            'is_paginated': paginator.pages > 1,
            'results_per_page': paginate_by,
            'has_next': paginator.has_next_page(page - 1),
            'has_previous': paginator.has_previous_page(page - 1),
            'page': page,
            'next': page + 1,
            'previous': page - 1,
            'last_on_page': paginator.last_on_page(page - 1),
            'first_on_page': paginator.first_on_page(page - 1),
            'pages': paginator.pages,
            'hits' : paginator.hits,

            'year_list' : [ (d, d.strftime(DATE_REPR["year"]), d.strftime(YEAR_URLS[date_kind])) for d in base_qset.dates('publish_from', "year", order='DESC') ],
            'date_list' : date_kind != 'detail' and [ (d, d.strftime(DATE_REPR[date_kind]), d.strftime(DATE_URLS[date_kind])) for d in qset.dates('publish_from', date_kind, order='DESC') ] or None,
            'current_date' : current_date,
            'current_date_text' : current_date.strftime(CURRENT_DATE_REPR[date_kind]),
            'date_kind' : date_kind,

            'content_type' : content_type,
            'listings' : listings,
            'category' : cat,
}, context_instance=RequestContext(request))

DATE_URLS = {'year' : '../%Y/', 'month' : '../../%Y/%m/', 'day' : '../../../%Y/%m/%d/',}
YEAR_URLS = {'year' : '../%Y/', 'month' : '../../%Y/', 'day' : '../../../%Y/', 'detail' : '../../../%Y/'}
DATE_REPR = {'year' : '%Y', 'month' : '%m/%Y', 'day' : '%d/%m/%Y', 'detail' : '%d/%m/%Y',}
CURRENT_DATE_REPR = {'year' : '', 'month' : '%Y', 'day' : '%m/%Y', 'detail' : '%d/%m/%Y',}

def home(request):
    """
    Homepage of the actual site.

    Params:
        request - HttpRequest from Django

    Raise:
        Http404 if there is no base category
    """
    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    return render_to_response(
            (
                # NEW
                'page/category/%s/category.html' % (cat.slug),
                'page/category.html',

                # OLD
                'category/%s/detail.html' % (cat.slug),
                'core/category_detail.html',
),
            {
                'category' : cat,
},
            context_instance=RequestContext(request)
)

def category_detail(request, category):
    """
    Homepage of a given category.

    Params:
        request - HttpRequest from Django
        category - category's ``tree_path``

    Raise:
        Http404 if the category doesn't exist
    """
    cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    return render_to_response(
            (
                # NEW
                'page/category/%s/category.html' % (cat.slug),
                'page/category.html',

                # OLD
                'category/%s/detail.html' % (cat.slug),
                'core/category_detail.html',
),
            {
                'category' : cat
},
            context_instance=RequestContext(request)
)

def export_test(*args, **kwargs):
    return []

def export_key(*args, **kwargs):
    return 'export:%d:%d' % (settings.SITE_ID, kwargs.get('count', 0))

@cache_this(export_key, export_test, timeout=60*60)
def export(request, count, models=None):
    if models is None:
        from ella.articles.models import Article
        models = [ Article, ]

    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    listing = Listing.objects.get_listing(count=count, category=cat, mods=models)
    return render_to_response(
            ('page/export_banner.html', 'export_banner.html',),
            {'category' : cat, 'listing' : listing},
            context_instance=RequestContext(request)
)

def page_not_found(request):
    response = render_to_response('page/404.html', {}, context_instance=RequestContext(request))
    response.status_code = 404
    return response

def handle_error(request):
    response = render_to_response('page/500.html', {}, context_instance=RequestContext(request))
    response.status_code = 500
    return response
<|endoftext|>


<commit_before>import itertools

def generate(env):
    """Add Builders and construction variables to the Environment."""

    if not 'boost_python' in env['TOOLS'][:-1]:
        env.Tool('system')
        env.Tool('textfile')
        env.AppendUnique(LIBS = ['boost_python'])
        env.AppendUnique(CPPDEFINES = ['BOOST_PYTHON_DYNAMIC_LIB',
                                       'BOOST_ALL_NO_LIB'])
        def BuildBoostPython(env, target, sources):
            # Code to build "target" from "source"

            SYSTEM = env['SYSTEM']
            if not SYSTEM == 'win':
                target += '.so'
            else:
                target += '.pyd'
            target = env.File(target).srcnode()
            targets = list(itertools.chain(*[env.SharedObject(None, source) for source in sources  if source.suffix in ['.cpp', '.cxx', '.c++']]))
            sources = [source for source in sources if source.suffix == '.h']
            if len(sources) == 1 and not SYSTEM == 'win':
                cmd = env.subst('$CXX') + ' -o $TARGET -x c++-header -c -fPIC ' + env.subst('$SHCXXFLAGS $_CCCOMCOM').replace('-x c++', '') + ' $SOURCE'
                if SYSTEM == 'linux':
                    cmd = env.Command(sources[0].target_from_source('', '.h.gch'), sources[0], cmd)
                else:
                    cmd = env.Command(sources[0].target_from_source('', '.h.pch'), sources[0], cmd)
                env.Depends(targets, cmd)
                if SYSTEM == 'osx':
                    env['CXX'] += " -include " + sources[0].target_from_source('', '.h').abspath
            env.Depends(target, targets)
            response = env.Textfile('response_file.rsp',
                         [tgt.abspath.replace('\\','/') for tgt in targets],
                         LINESEPARATOR=" ")
            env.Depends(target, response)
            env.Append(LINKFLAGS = '@' + response[0].abspath)
            if SYSTEM == 'win':
                return env.SharedLibrary(target, [])
            elif SYSTEM == 'osx':
                return env.LoadableModule(target, [],
                                          SHLINKFLAGS='$LINKFLAGS -bundle',
                                          FRAMEWORKSFLAGS='-flat_namespace -undefined suppress')
            else:
                return env.LoadableModule(target, [])

        env.AddMethod(BuildBoostPython)
        env.Tool('python')

def exists(env):
    return 1
<commit_msg>Update boost_python.py<commit_after>import itertools

def generate(env):
    """Add Builders and construction variables to the Environment."""

    if not 'boost_python' in env['TOOLS'][:-1]:
        env.Tool('system')
        env.Tool('textfile')
        env.AppendUnique(LIBS = ['boost_python'])
        env.AppendUnique(CPPDEFINES = ['BOOST_PYTHON_DYNAMIC_LIB',
                                       'BOOST_ALL_NO_LIB'])
        def BuildBoostPython(env, target, sources):
            # Code to build "target" from "source"

            SYSTEM = env['SYSTEM']
            if not SYSTEM == 'win':
                target += '.so'
            target = env.File(target).srcnode()
            targets = list(itertools.chain(*[env.SharedObject(None, source) for source in sources  if source.suffix in ['.cpp', '.cxx', '.c++']]))
            sources = [source for source in sources if source.suffix == '.h']
            if len(sources) == 1 and not SYSTEM == 'win':
                cmd = env.subst('$CXX') + ' -o $TARGET -x c++-header -c -fPIC ' + env.subst('$SHCXXFLAGS $_CCCOMCOM').replace('-x c++', '') + ' $SOURCE'
                if SYSTEM == 'linux':
                    cmd = env.Command(sources[0].target_from_source('', '.h.gch'), sources[0], cmd)
                else:
                    cmd = env.Command(sources[0].target_from_source('', '.h.pch'), sources[0], cmd)
                env.Depends(targets, cmd)
                if SYSTEM == 'osx':
                    env['CXX'] += " -include " + sources[0].target_from_source('', '.h').abspath
            env.Depends(target, targets)
            response = env.Textfile('response_file.rsp',
                         [tgt.abspath.replace('\\','/') for tgt in targets],
                         LINESEPARATOR=" ")
            env.Depends(target, response)
            env.Append(LINKFLAGS = '@' + response[0].abspath)
            if SYSTEM == 'win':
                return env.SharedLibrary(target, [], SHLIBPREFIX='',
                                                     SHLIBSUFFIX = '.pyd')
            elif SYSTEM == 'osx':
                return env.LoadableModule(target, [],
                                          SHLINKFLAGS='$LINKFLAGS -bundle',
                                          FRAMEWORKSFLAGS='-flat_namespace -undefined suppress')
            else:
                return env.LoadableModule(target, [])

        env.AddMethod(BuildBoostPython)
        env.Tool('python')

def exists(env):
    return 1
<|endoftext|>


<commit_before>import unittest
from isitdown.routes import is_spam

class TestUtils(unittest.TestCase):
    def test_isSpam(self):
        self.assertTrue(is_spam("google.it",["google.it"]))
        self.assertFalse(is_spam("google.it", ["google.com"]))
        self.assertTrue(is_spam("google",["google.it"]))
<commit_msg>Fix test<commit_after>import unittest
from isitdown.routes import is_spam

class TestUtils(unittest.TestCase):
    def test_isSpam(self):
        self.assertTrue(is_spam("google.it",["google.it"]))
        self.assertFalse(is_spam("google.it", ["google.com"]))
        self.assertTrue(is_spam("google.it",["google"]))
<|endoftext|>