<commit_before>from itertools import chain

from .. import errors

class Normalizer:

    def check_consistency_and_normalize(self, labels):
        self.check_label_consistency(labels)
        return [self.normalize(l) for l in labels]

    def normalize(self, label):
        raise NotImplementedError()

    def denormalize(self, normalized_label):
        raise NotImplementedError()

class ClassVerifier(Normalizer):

    def __init__(self, possible_labels):
        self.label_set = set(possible_labels)

    def check_label_consistency(self, labels):
        unique_labels = set(labels)
        if unique_labels - self.label_set:
            raise errors.ModelConsistencyError(
                "Labels {0} not in expected labels {1}"
                .format(unique_labels - self.label_set, self.label_set))
        elif self.label_set - unique_labels:
            raise errors.ModelConsistencyError(
                "Expected labels {0} represented"
                .format(self.label_set - unique_labels))

    def normalize(self, label):
        if label not in self.label_set:
            raise errors.ModelConsistencyError(
                "Label {0} not in list of expected labels {1}"
                .format(label, self.label_set))
        else:
            return label

    def denormalize(self, label):
        return label

class Binarizer(ClassVerifier):

    def __init__(self, possible_labels=None):
        super().__init__(possible_labels)
        self.possible_labels = possible_labels
        if possible_labels:
            self.label_index_map = \
                {l: i for i, l in enumerate(possible_labels)}

    def check_label_consistency(self, labels):
        unique_labels = set(chain(*(l for l in labels)))
        super().check_label_consistency(unique_labels)

    def normalize(self, label_set):
        binary_map = [0 for i in range(len(self.possible_labels))]
        for l in label_set:
            try:
                index = self.label_index_map[l]
            except KeyError:
                raise errors.ModelConsistencyError(
                    "Label {0} not in list of expected labels {1}"
                    .format(l, self.possible_labels))
            binary_map[index] = 1
        return binary_map

    def denormalize(self, binary_map):
        label_set = []
        for index, val in enumerate(binary_map):
            if val:
                label_set.append(self.possible_labels[index])

        return label_set
<commit_msg>Remove None as default value of possible_labels<commit_after>from itertools import chain

from .. import errors

class Normalizer:

    def check_consistency_and_normalize(self, labels):
        self.check_label_consistency(labels)
        return [self.normalize(l) for l in labels]

    def normalize(self, label):
        raise NotImplementedError()

    def denormalize(self, normalized_label):
        raise NotImplementedError()

class ClassVerifier(Normalizer):

    def __init__(self, possible_labels):
        self.label_set = set(possible_labels)

    def check_label_consistency(self, labels):
        unique_labels = set(labels)
        if unique_labels - self.label_set:
            raise errors.ModelConsistencyError(
                "Labels {0} not in expected labels {1}"
                .format(unique_labels - self.label_set, self.label_set))
        elif self.label_set - unique_labels:
            raise errors.ModelConsistencyError(
                "Expected labels {0} represented"
                .format(self.label_set - unique_labels))

    def normalize(self, label):
        if label not in self.label_set:
            raise errors.ModelConsistencyError(
                "Label {0} not in list of expected labels {1}"
                .format(label, self.label_set))
        else:
            return label

    def denormalize(self, label):
        return label

class Binarizer(ClassVerifier):

    def __init__(self, possible_labels):
        super().__init__(possible_labels)
        self.possible_labels = possible_labels
        if possible_labels:
            self.label_index_map = \
                {l: i for i, l in enumerate(possible_labels)}

    def check_label_consistency(self, labels):
        unique_labels = set(chain(*(l for l in labels)))
        super().check_label_consistency(unique_labels)

    def normalize(self, label_set):
        binary_map = [0 for i in range(len(self.possible_labels))]
        for l in label_set:
            try:
                index = self.label_index_map[l]
            except KeyError:
                raise errors.ModelConsistencyError(
                    "Label {0} not in list of expected labels {1}"
                    .format(l, self.possible_labels))
            binary_map[index] = 1
        return binary_map

    def denormalize(self, binary_map):
        label_set = []
        for index, val in enumerate(binary_map):
            if val:
                label_set.append(self.possible_labels[index])

        return label_set
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

__author__ = "Telmo Menezes (telmo@telmomenezes.com)"
__date__ = "Oct 2011"

import random
import math
from syn.core import *
from syn.drmap import *

class Evo:
    def __init__(self, targ_net, mrate=0.1, rrate=0.3, pop=500, tournament=2):
        self.targ_net = targ_net
        self.mrate = mrate
        self.rrate = rrate
        self.tournament = tournament
        self.pop = pop
        self.population = []

        self.syn_net = targ_net.load_net()
        self.nodes = net_node_count(self.syn_net)
        self.edges = net_edge_count(self.syn_net)
        self.nodes = 250
        self.edges = self.nodes * (net_edge_count(self.syn_net) / net_node_count(self.syn_net))
        self.max_cycles = self.edges * 100

        seed_random()

    def __del__(self):
        for i in range(self.pop):
            destroy_gpgenerator(self.population[i])
        destroy_net(self.syn_net)

    def run(self):
        bins = 10
        draw_drmap(self.syn_net, 'target.png', bins=bins)

        print 'Evolving gpgenerator'
        print 'Nodes:', self.nodes
        print 'Edges:', self.edges
        print 'Population:', self.pop
        print 'Mutation rate:', self.mrate
        print 'Tournament:', self.tournament
        print 'Recombination rate:', self.rrate

        # init population
        self.population = []
        self.fitness = []
        for i in range(self.pop):
            gen = create_gpgenerator()
            self.population.append(gen)
            self.fitness.append(0)

        print 'Population initialized.'

        cycle = 0
        best_fit = 9999999
        # evolutionary loop
        while True:
            # eval fitness
            for i in range(self.pop):
                gen = self.population[i]
                net = gpgen_run(gen, self.nodes, self.edges, self.max_cycles)

                compute_pageranks(self.syn_net)
                compute_pageranks(net)

                drmap1 = get_drmap_with_limits(self.syn_net, bins, -7.0, 7.0, -7.0, 7.0)
                drmap_log_scale(drmap1)
                drmap_normalize(drmap1)
                #drmap_binary(drmap1)

                drmap2 = get_drmap_with_limits(net, bins, -7.0, 7.0, -7.0, 7.0)
                drmap_log_scale(drmap2)
                drmap_normalize(drmap2)
                #drmap_binary(drmap2)

                fit = drmap_emd_dist(drmap1, drmap2)

                fit = 0
                for x in range(0, bins):
                    for y in range(0, bins):
                        wx = math.fabs(x - 5)
                        wy = math.fabs(y - 5)
                        w = wx * wy
                        fit -= drmap_get_value(drmap2, x, y) * w

                destroy_drmap(drmap1)
                destroy_drmap(drmap2)
                self.fitness[i] = fit
                print i, fit
                if fit < best_fit:
                    best_fit = fit
                    print_gpgen(self.population[i])
                    draw_drmap(net, 'best.png', bins=bins)
                destroy_net(net)

            print 'Generation %d => best fitness: %f' % (cycle, best_fit)
            cycle += 1

            # next generation
            newgen = []

            for i in range(self.pop):
                parent1 = -1
                for i in range(self.tournament):
                    parent1 = random.randint(0, self.pop - 1)
                    if (parent1 < 0) or (self.fitness[i] < self.fitness[parent1]):
                        parent1 = i

                child = None
                # recombine or clone
                if random.uniform(0, 1) < self.rrate:
                    parent2 = -1
                    for i in range(self.tournament):
                        parent2 = random.randint(0, self.pop - 1)
                        if (parent2 < 0) or (self.fitness[i] < self.fitness[parent2]):
                            parent2 = i

                    child = recombine_gpgens(self.population[parent1], self.population[parent2])
                else:
                    child = clone_gpgenerator(self.population[parent1])

                # mutate
                if random.uniform(0, 1) < self.mrate:
                    chicken = create_gpgenerator()
                    child2 = recombine_gpgens(child, chicken)
                    destroy_gpgenerator(chicken)
                    destroy_gpgenerator(child)
                    child = child2

                newgen.append(child)

            # replace generations
            for i in range(self.pop):
                destroy_gpgenerator(self.population[i])
            self.population = newgen
<commit_msg>tweaking evo algorithm<commit_after>#!/usr/bin/env python
# encoding: utf-8

__author__ = "Telmo Menezes (telmo@telmomenezes.com)"
__date__ = "Oct 2011"

import random
import math
from syn.core import *
from syn.drmap import *

class Evo:
    def __init__(self, targ_net, mrate=0.1, rrate=0.3, pop=500, tournament=2):
        self.targ_net = targ_net
        self.mrate = mrate
        self.rrate = rrate
        self.tournament = tournament
        self.pop = pop
        self.population = []

        self.syn_net = targ_net.load_net()
        self.nodes = net_node_count(self.syn_net)
        self.edges = net_edge_count(self.syn_net)
        self.nodes = 250
        self.edges = self.nodes * (net_edge_count(self.syn_net) / net_node_count(self.syn_net))
        self.max_cycles = self.edges * 100

        seed_random()

    def __del__(self):
        for i in range(self.pop):
            destroy_gpgenerator(self.population[i])
        destroy_net(self.syn_net)

    def run(self):
        bins = 10
        draw_drmap(self.syn_net, 'target.png', bins=bins)

        print 'Evolving gpgenerator'
        print 'Nodes:', self.nodes
        print 'Edges:', self.edges
        print 'Population:', self.pop
        print 'Mutation rate:', self.mrate
        print 'Tournament:', self.tournament
        print 'Recombination rate:', self.rrate

        # init population
        self.population = []
        self.fitness = []
        for i in range(self.pop):
            gen = create_gpgenerator()
            self.population.append(gen)
            self.fitness.append(0)

        print 'Population initialized.'

        cycle = 0
        best_fit = 9999999
        # evolutionary loop
        while True:
            # eval fitness
            for i in range(self.pop):
                gen = self.population[i]
                net = gpgen_run(gen, self.nodes, self.edges, self.max_cycles)

                compute_pageranks(self.syn_net)
                compute_pageranks(net)

                drmap1 = get_drmap_with_limits(self.syn_net, bins, -7.0, 7.0, -7.0, 7.0)
                drmap_log_scale(drmap1)
                drmap_normalize(drmap1)
                #drmap_binary(drmap1)

                drmap2 = get_drmap_with_limits(net, bins, -7.0, 7.0, -7.0, 7.0)
                drmap_log_scale(drmap2)
                drmap_normalize(drmap2)
                #drmap_binary(drmap2)

                fit = drmap_emd_dist(drmap1, drmap2)

                #fit = 0
                #for x in range(0, bins):
                #    for y in range(0, bins):
                #        wx = math.fabs(x - 5)
                #        wy = math.fabs(y - 5)
                #        w = wx * wy
                #        fit -= drmap_get_value(drmap2, x, y) * w

                destroy_drmap(drmap1)
                destroy_drmap(drmap2)
                self.fitness[i] = fit
                print i, fit
                if fit < best_fit:
                    best_fit = fit
                    print_gpgen(self.population[i])
                    draw_drmap(net, 'best.png', bins=bins)
                destroy_net(net)

            print 'Generation %d => best fitness: %f' % (cycle, best_fit)
            cycle += 1

            # next generation
            newgen = []

            for i in range(self.pop):
                parent1 = -1
                for i in range(self.tournament):
                    parent1 = random.randint(0, self.pop - 1)
                    if (parent1 < 0) or (self.fitness[i] < self.fitness[parent1]):
                        parent1 = i

                child = None
                # recombine or clone
                if random.uniform(0, 1) < self.rrate:
                    parent2 = -1
                    for i in range(self.tournament):
                        parent2 = random.randint(0, self.pop - 1)
                        if (parent2 < 0) or (self.fitness[i] < self.fitness[parent2]):
                            parent2 = i

                    child = recombine_gpgens(self.population[parent1], self.population[parent2])
                else:
                    child = clone_gpgenerator(self.population[parent1])

                # mutate
                if random.uniform(0, 1) < self.mrate:
                    chicken = create_gpgenerator()
                    child2 = recombine_gpgens(child, chicken)
                    destroy_gpgenerator(chicken)
                    destroy_gpgenerator(child)
                    child = child2

                newgen.append(child)

            # replace generations
            for i in range(self.pop):
                destroy_gpgenerator(self.population[i])
            self.population = newgen
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Performs all git-svn setup steps necessary for 'git svn dcommit' to work.

Assumes that trunk of the svn remote maps to master of the git remote.

Example:
git clone https://chromium.googlesource.com/chromium/tools/depot_tools
cd depot_tools
git auto-svn
"""

import argparse
import os
import sys
import urlparse

import subprocess2

from git_common import run as run_git
from git_common import run_stream as run_git_stream
from git_common import set_config, root, ROOT
from git_footers import parse_footers, get_unique, GIT_SVN_ID_PATTERN

SVN_EXE = ROOT+'\\svn.bat' if sys.platform.startswith('win') else 'svn'

def run_svn(*cmd, **kwargs):
  """Runs an svn command.

  Returns (stdout, stderr) as a pair of strings.

  Raises subprocess2.CalledProcessError on nonzero return code.
  """
  kwargs.setdefault('stdin', subprocess2.PIPE)
  kwargs.setdefault('stdout', subprocess2.PIPE)
  kwargs.setdefault('stderr', subprocess2.PIPE)

  cmd = (SVN_EXE,) + cmd
  proc = subprocess2.Popen(cmd, **kwargs)
  ret, err = proc.communicate()
  retcode = proc.wait()
  if retcode != 0:
    raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err)

  return ret, err

def main(argv):
  # No command line flags. Just use the parser to prevent people from trying
  # to pass flags that don't do anything, and to provide 'usage'.
  parser = argparse.ArgumentParser(
      description='Automatically set up git-svn for a repo mirrored from svn.')
  parser.parse_args(argv[1:])

  upstream = root()
  message = run_git('log', '-1', '--format=%B', upstream)
  footers = parse_footers(message)
  git_svn_id = get_unique(footers, 'git-svn-id')
  match = GIT_SVN_ID_PATTERN.match(git_svn_id)
  assert match, 'No valid git-svn-id footer found on %s.' % upstream
  print 'Found git-svn-id footer %s on %s' % (match.group(1), upstream)

  parsed_svn = urlparse.urlparse(match.group(1))
  path_components = parsed_svn.path.split('/')
  svn_repo = None
  svn_path = None
  for i in xrange(len(path_components)):
    try:
      maybe_repo = '%s://%s%s' % (
          parsed_svn.scheme, parsed_svn.netloc, '/'.join(path_components[:i+1]))
      print 'Checking ', maybe_repo
      run_svn('info', maybe_repo)
      svn_repo = maybe_repo
      svn_path = '/'.join(path_components[i+1:])
      break
    except subprocess2.CalledProcessError:
      continue
  assert svn_repo is not None, 'Unable to find svn repo for %s' % match.group(1)
  print 'Found upstream svn repo %s and path %s' % (svn_repo, svn_path)

  set_config('svn-remote.svn.url', svn_repo)
  set_config('svn-remote.svn.fetch',
             '%s:refs/remotes/%s' % (svn_path, upstream))
  print 'Configured metadata, running "git svn fetch". This may take some time.'
  for line in run_git_stream('svn', 'fetch').xreadlines():
    print line.strip()

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>Fail noisily when git auto-svn can't authenticate.<commit_after>#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Performs all git-svn setup steps necessary for 'git svn dcommit' to work.

Assumes that trunk of the svn remote maps to master of the git remote.

Example:
git clone https://chromium.googlesource.com/chromium/tools/depot_tools
cd depot_tools
git auto-svn
"""

import argparse
import os
import sys
import urlparse

import subprocess2

from git_common import run as run_git
from git_common import run_stream as run_git_stream
from git_common import set_config, root, ROOT
from git_footers import parse_footers, get_unique, GIT_SVN_ID_PATTERN

SVN_EXE = ROOT+'\\svn.bat' if sys.platform.startswith('win') else 'svn'

def run_svn(*cmd, **kwargs):
  """Runs an svn command.

  Returns (stdout, stderr) as a pair of strings.

  Raises subprocess2.CalledProcessError on nonzero return code.
  """
  kwargs.setdefault('stdin', subprocess2.PIPE)
  kwargs.setdefault('stdout', subprocess2.PIPE)
  kwargs.setdefault('stderr', subprocess2.PIPE)

  cmd = (SVN_EXE,) + cmd
  proc = subprocess2.Popen(cmd, **kwargs)
  ret, err = proc.communicate()
  retcode = proc.wait()
  if retcode != 0:
    raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err)

  return ret, err

def main(argv):
  # No command line flags. Just use the parser to prevent people from trying
  # to pass flags that don't do anything, and to provide 'usage'.
  parser = argparse.ArgumentParser(
      description='Automatically set up git-svn for a repo mirrored from svn.')
  parser.parse_args(argv[1:])

  upstream = root()
  message = run_git('log', '-1', '--format=%B', upstream)
  footers = parse_footers(message)
  git_svn_id = get_unique(footers, 'git-svn-id')
  match = GIT_SVN_ID_PATTERN.match(git_svn_id)
  assert match, 'No valid git-svn-id footer found on %s.' % upstream
  print 'Found git-svn-id footer %s on %s' % (match.group(1), upstream)

  parsed_svn = urlparse.urlparse(match.group(1))
  path_components = parsed_svn.path.split('/')
  svn_repo = None
  svn_path = None
  for i in xrange(len(path_components)):
    try:
      maybe_repo = '%s://%s%s' % (
          parsed_svn.scheme, parsed_svn.netloc, '/'.join(path_components[:i+1]))
      print 'Checking ', maybe_repo
      run_svn('info', maybe_repo)
      svn_repo = maybe_repo
      svn_path = '/'.join(path_components[i+1:])
      break
    except subprocess2.CalledProcessError, e:
      if 'E170001' in str(e):
        print 'Authentication failed:'
        print e
        print ('Try running "svn ls %s" with the password'
               ' from https://chromium-access.appspot.com' % maybe_repo)
        print
      continue
  assert svn_repo is not None, 'Unable to find svn repo for %s' % match.group(1)
  print 'Found upstream svn repo %s and path %s' % (svn_repo, svn_path)

  set_config('svn-remote.svn.url', svn_repo)
  set_config('svn-remote.svn.fetch',
             '%s:refs/remotes/%s' % (svn_path, upstream))
  print 'Configured metadata, running "git svn fetch". This may take some time.'
  for line in run_git_stream('svn', 'fetch').xreadlines():
    print line.strip()

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>"""Scheduler for running via docker."""
import time

import docker
try:
    from pprintpp import pprint
except ImportError:
    from pprint import pprint

from rickshaw.scheduler import Scheduler

class DockerScheduler(Scheduler):
    """A base docker scheduler"""

    def __init__(self, debug=False, **kwargs):
        self.client = docker.client.from_env()
        self.cyclus_container = None
        self.server_tag = "ergs/cyclus-server-dev"
        if debug:
            self.server_cmd = "--debug"
        else:
            self.server_cmd = ""
        self.cyclus_server_name = "rickshaw_metadata_server"
        self.cyclus_server_host = None
        self.cyclus_server_ready = False
        self.gathered_annotations = False
        self._have_swarm = False
        self._find_ncpu()

    def _find_ncpu(self):
        try:
            # get NCPUs for swarm
            ncpu = 0.0
            print('try')
            for node in self.client.nodes.list():
                ncpu += node.attrs['Description']['Resources']['NanoCPUs'] * 1e-9
            self._have_swarm = True
        except docker.errors.APIError:
            print('except')
            # get NCPUs for local host
            ncpu = self.client.info()['NCPU']
            self._have_swarm = False
        self.ncpu = int(ncpu)

    def __del__(self):
        self.stop_cyclus_server()

    def start_cyclus_server(self):
        """Starts up a cyclus server at a remote location."""
        print("starting cyclus server")
        cc = self.cyclus_container = self.client.containers.run(self.server_tag,
                                                                self.server_cmd,
                                        ports={'4242/tcp': ('127.0.0.1', 4242)},
                                                   name=self.cyclus_server_name,
                                                         publish_all_ports=True,
                                                                    detach=True)
        host = self.client.networks.get('bridge').attrs['Containers'][cc.id]['IPv4Address']
        if '/' in host:
            self.cyclus_server_host, _, _ = host.rpartition('/')
        else:
            self.cyclus_server_host = host
        print("cyclus server started at host " + host)
        time.sleep(3)
        self.cyclus_server_ready = True
        for line in cc.logs(stream=True):
            print('[cyclus] ' + line.decode(), end='')

    def stop_cyclus_server(self):
        """Stops the cyclus server running in a remote location"""
        if self.cyclus_container is not None:
            self.cyclus_container.stop()
            self.cyclus_container = None
        self.cyclus_server_ready = False

    def queue(self):
        """Obtains the current queue status and retuns the jobs that are scheduled
        and status of each job.
        """
        return [(c.id, c.status) for c in self.client.containers.list()]

    def schedule(self, sim):
        """Schedules a simulation to be executed."""
        #print("would have scheduled sim: ", repr(sim))

    def want_n_more_jobs(self):
        """Determine how many more new jobs to schedule."""
        n = self.ncpu*2 - len(self.queue())
        print("will want, " + str(n))
        pprint(self.client.swarm.attrs)
        return n
<commit_msg>update<commit_after>"""Scheduler for running via docker."""
import time

import docker
try:
    from pprintpp import pprint
except ImportError:
    from pprint import pprint

from rickshaw.scheduler import Scheduler

class DockerScheduler(Scheduler):
    """A base docker scheduler"""

    def __init__(self, debug=False, **kwargs):
        self.client = docker.from_env()
        self.cyclus_container = None
        self.server_tag = "ergs/cyclus-server-dev"
        if debug:
            self.server_cmd = "--debug"
        else:
            self.server_cmd = ""
        self.cyclus_server_name = "rickshaw_metadata_server"
        self.cyclus_server_host = None
        self.cyclus_server_ready = False
        self.gathered_annotations = False
        self._have_swarm = False
        self._find_ncpu()

    def _find_ncpu(self):
        try:
            # get NCPUs for swarm
            ncpu = 0.0
            print('try')
            for node in self.nodes.list():
                ncpu += node.attrs['Description']['Resources']['NanoCPUs'] * 1e-9
            self._have_swarm = True
        except docker.errors.APIError:
            print('except')
            # get NCPUs for local host
            ncpu = self.info()['NCPU']
            self._have_swarm = False
        self.ncpu = int(ncpu)

    def __del__(self):
        self.stop_cyclus_server()

    def start_cyclus_server(self):
        """Starts up a cyclus server at a remote location."""
        print("starting cyclus server")
        cc = self.cyclus_container = self.client.containers.run(self.server_tag,
                                                                self.server_cmd,
                                        ports={'4242/tcp': ('127.0.0.1', 4242)},
                                                   name=self.cyclus_server_name,
                                                         publish_all_ports=True,
                                                                    detach=True)
        host = self.client.networks.get('bridge').attrs['Containers'][cc.id]['IPv4Address']
        if '/' in host:
            self.cyclus_server_host, _, _ = host.rpartition('/')
        else:
            self.cyclus_server_host = host
        print("cyclus server started at host " + host)
        time.sleep(3)
        self.cyclus_server_ready = True
        for line in cc.logs(stream=True):
            print('[cyclus] ' + line.decode(), end='')

    def stop_cyclus_server(self):
        """Stops the cyclus server running in a remote location"""
        if self.cyclus_container is not None:
            self.cyclus_container.stop()
            self.cyclus_container = None
        self.cyclus_server_ready = False

    def queue(self):
        """Obtains the current queue status and retuns the jobs that are scheduled
        and status of each job.
        """
        return [(c.id, c.status) for c in self.client.containers.list()]

    def schedule(self, sim):
        """Schedules a simulation to be executed."""
        #print("would have scheduled sim: ", repr(sim))

    def want_n_more_jobs(self):
        """Determine how many more new jobs to schedule."""
        n = self.ncpu*2 - len(self.queue())
        print("will want, " + str(n))
        pprint(self.client.swarm.attrs)
        return n
<|endoftext|>


<commit_before>from binascii import hexlify, unhexlify
from contextlib import closing
import os
from pathlib import Path
import shlex
import subprocess
import sys
import uuid

from future.utils import raise_from
import logbook
from sqlacfg import Config
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
import trollius as asyncio
from trollius import From

from .migration import get_upgrade_path
from .model import Base, User, PublicKey, ConfigSetting
from .util import block_update, sanitize_path
from .exc import (UserNotFoundError, KeyNotFoundError, PermissionDenied,
                  NoSuchRepository, GitHomeError)

log = logbook.Logger('githome')

class GitHome(object):
    REPOS_PATH = 'repos'
    DB_PATH = 'githome.sqlite'

    @property
    def dsn(self):
        return 'sqlite:///{}'.format(self.path / self.DB_PATH)

    def __init__(self, path):
        self.path = Path(path)
        self.bind = create_engine(self.dsn)
        self.session = scoped_session(sessionmaker(bind=self.bind))
        self.config = Config(ConfigSetting, self.session)
        self._update_authkeys = False

    def save(self):
        self.session.commit()

        if self._update_authkeys:
            self._update_authkeys = False

            if not self.config['local']['update_authorized_keys']:
                log.info('Not updating authorized_keys, disabled in config')
            else:
                self.update_authorized_keys()

    def create_user(self, name):
        user = User(name=name)
        self.session.add(user)
        return user

    def delete_user(self, name):
        user = self.get_user_by_name(name)
        self.session.delete(user)
        self._update_authkeys = True

        return True

    def iter_users(self, order_by=User.name):
        qry = self.session.query(User)
        if order_by:
            qry = qry.order_by(order_by)

        return qry

    def add_key(self, user, pkey):
        try:
            self.get_key_by_fingerprint(pkey.fingerprint)
        except KeyNotFoundError:
            key = PublicKey.from_pkey(pkey)
        else:
            raise GitHomeError('Key {} already in database'.format(
                pkey.readable_fingerprint
            ))

        key.user = user
        self.session.add(key)

        self._update_authkeys = True
        return pkey

    def delete_key(self, fingerprint):
        self.session.delete(self.get_key_by_fingerprint(fingerprint))
        self._update_authkeys = True

    def get_repo(self, rel_path, create=False):
        path = self.path / self.REPOS_PATH / rel_path

        if not path.exists() or not path.is_dir():
            if create:
                # create the repo
                path.mkdir(parents=True)
                subprocess.check_call([
                    'git', 'init', '--quiet', '--bare',
                    '--shared=0600', str(path),
                ])
            else:
                raise NoSuchRepository('Repository {} no found and not '
                                       'creating.'.format(rel_path))
        return path.absolute()

    def get_user_by_name(self, name):
        try:
            return self.session.query(User).filter_by(name=name.lower()).one()
        except NoResultFound as e:
            raise_from(UserNotFoundError('User {} not found'.format(name)), e)

    def get_key_by_fingerprint(self, fingerprint):
        try:
            return (self.session.query(PublicKey)
                                .filter_by(fingerprint=hexlify(fingerprint))
                                .one())
        except NoResultFound as e:
            raise_from(KeyNotFoundError('Key {} not found'.format(hexlify
                       (fingerprint))), e)

    def get_authorized_keys_block(self):
        pkeys = []
        for key in self.session.query(PublicKey):
            if self.config['local']['use_gh_client']:
                pkey = key.as_pkey()
                spath = (self.path / self.config['local']['gh_client_socket'])
                args = [self.config['local']['gh_client_executable'],
                        str(spath.absolute()),
                        hexlify(pkey.fingerprint)]
            else:
                args = [
                    self.config['local']['githome_executable'],
                ]

                args.extend([
                    '--githome',
                    str(self.path.absolute()),
                    'shell',
                    key.user.name,
                ])

            full_cmd = ' '.join("'{}'".format(p) for p in args)

            opts = {
                'command': full_cmd,
                'no-agent-forwarding': True,
                'no-port-forwarding': True,
                'no-pty': True,
                'no-user-rc': True,
                'no-x11-forwarding': True,
            }
            pkey = key.as_pkey(options=opts)

            pkeys.append(pkey)

        return ''.join(pkey.to_pubkey_line() + '\n' for pkey in pkeys)

    def update_authorized_keys(self):
        ak = Path(self.config['local']['authorized_keys_file'])
        if not ak.exists():
            log.error('Refusing to update non-existant authorized_keys file: '
                      '{}'.format(ak))
            return

        start_marker = (self.config['local']['authorized_keys_start_marker']
                        .format(self.config['githome']['id']))
        end_marker = (self.config['local']['authorized_keys_start_marker']
                      .format(self.config['githome']['id']))

        old = ak.open().read()

        ak.open('wb').write(block_update(
            start_marker,
            end_marker,
            old,
            self.get_authorized_keys_block(),
        ))
        log.info('Updated {}'.format(ak))

    def authorize_command(self, user, command):
        CMD_WHITELIST = [
            'git-upload-pack',
            'git-receive-pack',
            'git-upload-archive',
        ]

        if not command[0] in CMD_WHITELIST:
            raise PermissionDenied(
                '{} is not a whitelisted command.'.format(command[0])
            )

        if len(command) < 2:
            raise PermissionDenied(
                'Missing repository parameter'
            )

        # FIXME: check user read rights to repository
        # FIXME: check if user may create repositories
        can_create = True

        repo_path = self.get_repo(sanitize_path(command[1]), create=can_create)
        # FIXME: if necessary, check write rights to repository

        if command[0] == 'git-upload-pack':
            return [command[0], '--strict',   # do not try /.git
                    str(repo_path)]
        elif command[0] == 'git-receive-pack':
            return [command[0], str(repo_path)]
        elif command[0] == 'git-upload-archive':
            return [command[0], str(repo_path)]
        else:
            raise GitHomeError(
                'Command {} is whitelisted, but not explicitly handled.'
                .format(command[0])
            )

    @classmethod
    def check(cls, path):
        """Check if a githome exists at path.

        :param path: A :class:`~pathlib.Path`.
        """
        return (path / cls.DB_PATH).exists()

    @classmethod
    def initialize(cls, path):
        """Initialize new githome at path.

        :param path: A :class:`~pathlib.Path`.
        :param initial_cfg:: Additional configuration settings.
        """
        # create paths
        (path / cls.REPOS_PATH).mkdir()

        # instantiate
        gh = cls(path)

        # create database
        Base.metadata.create_all(bind=gh.bind)

        # create initial configuration
        local = gh.config['local']
        local['update_authorized_keys'] = True
        local['authorized_keys_file'] = os.path.abspath(
            os.path.expanduser('~/.ssh/authorized_keys')
        )
        local['githome_executable'] = str(Path(sys.argv[0]).absolute())
        local['authorized_keys_start_marker'] = (
            '# -- added by githome {}, do not remove these markers --\n'
        )
        local['authorized_keys_end_marker'] = (
            '# -- end githome {}. keep trailing newline! --\n'
        )
        local['use_gh_client'] = True
        gh_client = str(Path(__file__).with_name('gh_client'))
        local['gh_client_executable'] = gh_client
        local['gh_client_socket'] = 'ghclient.sock'

        gh.config['githome']['id'] = str(uuid.uuid4())

        gh.save()

        return gh

    def __repr__(self):
        return '{0.__class__.__name__}(path={0.path!r})'.format(self)

    def upgrade_db(self):
        with self.bind.connect() as con, con.begin() as trans:
            for mfunc in get_upgrade_path(self):
                mfunc(trans)

    def run_server(self, debug=False):
        @asyncio.coroutine
        def gh_server():
            socket = str(self.path / self.config['local']['gh_client_socket'])

            log.info('Server socket: {}'.format(socket))
            if os.path.exists(socket):
                log.debug('Removing stale socket')
                os.unlink(socket)

            yield From(asyncio.start_unix_server(gh_proto, socket))

        @asyncio.coroutine
        def gh_proto(client_reader, client_writer):
            con_id = uuid.uuid4()
            log = logbook.Logger('client-{}'.format(con_id))
            log.debug('connected')

            with closing(client_writer._transport):
                keyfp = (yield From(client_reader.readline())).strip()

                if not keyfp:
                    log.warning('unexpected connection close')
                    return

                cmd = (yield From(client_reader.readline())).strip()
                log.debug('Read command: {!r}'.format(cmd))

                try:
                    key = self.get_key_by_fingerprint(unhexlify(keyfp))
                    user = key.user
                    log.info('authenticated as {}'.format(user.name))

                    # check if user is allowed to execute command
                    clean_command = self.authorize_command(user,
                                                           shlex.split(cmd))
                except Exception as e:
                    # deny on every exception, no exceptions!
                    log.warning('permission denied: {}'.format(e))
                    yield From(client_writer.write('E access denied\n'))
                    return
                else:
                    # wrapped in else, for defensive reasons
                    log.info('Authorized for {!r}'.format(clean_command))

                    # write OK byte
                    yield From(client_writer.write('OK\n'))

                    # actualy reply
                    for part in clean_command:
                        yield From(client_writer.write(part + '\n'))

        loop = asyncio.get_event_loop()

        # debug
        loop.set_debug(debug)

        # start server
        loop.run_until_complete(gh_server())

        try:
            loop.run_forever()
        finally:
            loop.close()
<commit_msg>Create alembic_version table with a version on newer initializations to make alembic work.<commit_after>from binascii import hexlify, unhexlify
from contextlib import closing
import os
from pathlib import Path
import shlex
import subprocess
import sys
import uuid

from future.utils import raise_from
import logbook
from sqlacfg import Config
from sqlalchemy import create_engine, Table, Column, String
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
import trollius as asyncio
from trollius import From

from .migration import get_upgrade_path
from .model import Base, User, PublicKey, ConfigSetting
from .util import block_update, sanitize_path
from .exc import (UserNotFoundError, KeyNotFoundError, PermissionDenied,
                  NoSuchRepository, GitHomeError)

log = logbook.Logger('githome')

class GitHome(object):
    REPOS_PATH = 'repos'
    DB_PATH = 'githome.sqlite'

    @property
    def dsn(self):
        return 'sqlite:///{}'.format(self.path / self.DB_PATH)

    def __init__(self, path):
        self.path = Path(path)
        self.bind = create_engine(self.dsn)
        self.session = scoped_session(sessionmaker(bind=self.bind))
        self.config = Config(ConfigSetting, self.session)
        self._update_authkeys = False

    def save(self):
        self.session.commit()

        if self._update_authkeys:
            self._update_authkeys = False

            if not self.config['local']['update_authorized_keys']:
                log.info('Not updating authorized_keys, disabled in config')
            else:
                self.update_authorized_keys()

    def create_user(self, name):
        user = User(name=name)
        self.session.add(user)
        return user

    def delete_user(self, name):
        user = self.get_user_by_name(name)
        self.session.delete(user)
        self._update_authkeys = True

        return True

    def iter_users(self, order_by=User.name):
        qry = self.session.query(User)
        if order_by:
            qry = qry.order_by(order_by)

        return qry

    def add_key(self, user, pkey):
        try:
            self.get_key_by_fingerprint(pkey.fingerprint)
        except KeyNotFoundError:
            key = PublicKey.from_pkey(pkey)
        else:
            raise GitHomeError('Key {} already in database'.format(
                pkey.readable_fingerprint
            ))

        key.user = user
        self.session.add(key)

        self._update_authkeys = True
        return pkey

    def delete_key(self, fingerprint):
        self.session.delete(self.get_key_by_fingerprint(fingerprint))
        self._update_authkeys = True

    def get_repo(self, rel_path, create=False):
        path = self.path / self.REPOS_PATH / rel_path

        if not path.exists() or not path.is_dir():
            if create:
                # create the repo
                path.mkdir(parents=True)
                subprocess.check_call([
                    'git', 'init', '--quiet', '--bare',
                    '--shared=0600', str(path),
                ])
            else:
                raise NoSuchRepository('Repository {} no found and not '
                                       'creating.'.format(rel_path))
        return path.absolute()

    def get_user_by_name(self, name):
        try:
            return self.session.query(User).filter_by(name=name.lower()).one()
        except NoResultFound as e:
            raise_from(UserNotFoundError('User {} not found'.format(name)), e)

    def get_key_by_fingerprint(self, fingerprint):
        try:
            return (self.session.query(PublicKey)
                                .filter_by(fingerprint=hexlify(fingerprint))
                                .one())
        except NoResultFound as e:
            raise_from(KeyNotFoundError('Key {} not found'.format(hexlify
                       (fingerprint))), e)

    def get_authorized_keys_block(self):
        pkeys = []
        for key in self.session.query(PublicKey):
            if self.config['local']['use_gh_client']:
                pkey = key.as_pkey()
                spath = (self.path / self.config['local']['gh_client_socket'])
                args = [self.config['local']['gh_client_executable'],
                        str(spath.absolute()),
                        hexlify(pkey.fingerprint)]
            else:
                args = [
                    self.config['local']['githome_executable'],
                ]

                args.extend([
                    '--githome',
                    str(self.path.absolute()),
                    'shell',
                    key.user.name,
                ])

            full_cmd = ' '.join("'{}'".format(p) for p in args)

            opts = {
                'command': full_cmd,
                'no-agent-forwarding': True,
                'no-port-forwarding': True,
                'no-pty': True,
                'no-user-rc': True,
                'no-x11-forwarding': True,
            }
            pkey = key.as_pkey(options=opts)

            pkeys.append(pkey)

        return ''.join(pkey.to_pubkey_line() + '\n' for pkey in pkeys)

    def update_authorized_keys(self):
        ak = Path(self.config['local']['authorized_keys_file'])
        if not ak.exists():
            log.error('Refusing to update non-existant authorized_keys file: '
                      '{}'.format(ak))
            return

        start_marker = (self.config['local']['authorized_keys_start_marker']
                        .format(self.config['githome']['id']))
        end_marker = (self.config['local']['authorized_keys_start_marker']
                      .format(self.config['githome']['id']))

        old = ak.open().read()

        ak.open('wb').write(block_update(
            start_marker,
            end_marker,
            old,
            self.get_authorized_keys_block(),
        ))
        log.info('Updated {}'.format(ak))

    def authorize_command(self, user, command):
        CMD_WHITELIST = [
            'git-upload-pack',
            'git-receive-pack',
            'git-upload-archive',
        ]

        if not command[0] in CMD_WHITELIST:
            raise PermissionDenied(
                '{} is not a whitelisted command.'.format(command[0])
            )

        if len(command) < 2:
            raise PermissionDenied(
                'Missing repository parameter'
            )

        # FIXME: check user read rights to repository
        # FIXME: check if user may create repositories
        can_create = True

        repo_path = self.get_repo(sanitize_path(command[1]), create=can_create)
        # FIXME: if necessary, check write rights to repository

        if command[0] == 'git-upload-pack':
            return [command[0], '--strict',   # do not try /.git
                    str(repo_path)]
        elif command[0] == 'git-receive-pack':
            return [command[0], str(repo_path)]
        elif command[0] == 'git-upload-archive':
            return [command[0], str(repo_path)]
        else:
            raise GitHomeError(
                'Command {} is whitelisted, but not explicitly handled.'
                .format(command[0])
            )

    @classmethod
    def check(cls, path):
        """Check if a githome exists at path.

        :param path: A :class:`~pathlib.Path`.
        """
        return (path / cls.DB_PATH).exists()

    @classmethod
    def initialize(cls, path):
        """Initialize new githome at path.

        :param path: A :class:`~pathlib.Path`.
        :param initial_cfg:: Additional configuration settings.
        """
        # create paths
        (path / cls.REPOS_PATH).mkdir()

        # instantiate
        gh = cls(path)

        # create database
        Base.metadata.create_all(bind=gh.bind)

        # create alembic metadata table
        avtable = Table('alembic_version', Base.metadata,
                        Column('version_num', String(32), nullable=False)
                        )
        avtable.create(bind=gh.bind)
        qry = (avtable.insert()
                      .values(version_num='4160ccb58402'))
        with gh.bind.begin() as con:
            con.execute(qry)

        # create initial configuration
        local = gh.config['local']
        local['update_authorized_keys'] = True
        local['authorized_keys_file'] = os.path.abspath(
            os.path.expanduser('~/.ssh/authorized_keys')
        )
        local['githome_executable'] = str(Path(sys.argv[0]).absolute())
        local['authorized_keys_start_marker'] = (
            '# -- added by githome {}, do not remove these markers --\n'
        )
        local['authorized_keys_end_marker'] = (
            '# -- end githome {}. keep trailing newline! --\n'
        )
        local['use_gh_client'] = True
        gh_client = str(Path(__file__).with_name('gh_client'))
        local['gh_client_executable'] = gh_client
        local['gh_client_socket'] = 'ghclient.sock'

        gh.config['githome']['id'] = str(uuid.uuid4())

        gh.save()

        return gh

    def __repr__(self):
        return '{0.__class__.__name__}(path={0.path!r})'.format(self)

    def upgrade_db(self):
        with self.bind.connect() as con, con.begin() as trans:
            for mfunc in get_upgrade_path(self):
                mfunc(trans)

    def run_server(self, debug=False):
        @asyncio.coroutine
        def gh_server():
            socket = str(self.path / self.config['local']['gh_client_socket'])

            log.info('Server socket: {}'.format(socket))
            if os.path.exists(socket):
                log.debug('Removing stale socket')
                os.unlink(socket)

            yield From(asyncio.start_unix_server(gh_proto, socket))

        @asyncio.coroutine
        def gh_proto(client_reader, client_writer):
            con_id = uuid.uuid4()
            log = logbook.Logger('client-{}'.format(con_id))
            log.debug('connected')

            with closing(client_writer._transport):
                keyfp = (yield From(client_reader.readline())).strip()

                if not keyfp:
                    log.warning('unexpected connection close')
                    return

                cmd = (yield From(client_reader.readline())).strip()
                log.debug('Read command: {!r}'.format(cmd))

                try:
                    key = self.get_key_by_fingerprint(unhexlify(keyfp))
                    user = key.user
                    log.info('authenticated as {}'.format(user.name))

                    # check if user is allowed to execute command
                    clean_command = self.authorize_command(user,
                                                           shlex.split(cmd))
                except Exception as e:
                    # deny on every exception, no exceptions!
                    log.warning('permission denied: {}'.format(e))
                    yield From(client_writer.write('E access denied\n'))
                    return
                else:
                    # wrapped in else, for defensive reasons
                    log.info('Authorized for {!r}'.format(clean_command))

                    # write OK byte
                    yield From(client_writer.write('OK\n'))

                    # actualy reply
                    for part in clean_command:
                        yield From(client_writer.write(part + '\n'))

        loop = asyncio.get_event_loop()

        # debug
        loop.set_debug(debug)

        # start server
        loop.run_until_complete(gh_server())

        try:
            loop.run_forever()
        finally:
            loop.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# coding=utf-8
# *******************************************************************
# *** GithubCloner ***
# * Description:
#   A script that clones public Github repositories
#   of users and organizations automatically.
# * Version:
#   v0.3
# * Homepage:
#   https://github.com/mazen160/GithubCloner
# * Author:
#   Mazin Ahmed <Mazin AT MazinAhmed DOT net>
# *******************************************************************

# Modules
import argparse
import git
import json
import os
import queue
import requests
import threading
import time

class getReposURLs(object):
    def __init__(self):
        self.user_agent = "GithubCloner (https://github.com/mazen160/GithubCloner)"
        self.headers = {'User-Agent': self.user_agent, 'Accept': '*/*'}
        self.timeout = 3

    def UserGists(self, user, username=None, token=None):
        """
        Returns a list of GIT URLs for accessible gists.
        Input:-
        user: Github user.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github gist repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/users/{0}/gists?page={1}".format(user, current_page)
            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            for i in range(len(resp)):
                URLs.append(resp[i]["git_pull_url"])
            current_page += 1
        return(URLs)

    def AuthenticatedGists(self, username, token):
        """
        Returns a list of gists of an authenticated user.
        Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github gist repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/gists?page={0}".format(current_page)
            resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)
            for i in range(len(resp)):
                URLs.append(resp[i]["git_pull_url"])
            current_page += 1

        return(URLs)

    def fromUser(self, user, username=None, token=None, include_gists=False):
        """
        Retrieves a list of repositories for a Github user.
        Input:-
        user: Github username.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/users/{0}/repos?per_page=40000000&page={1}".format(user, current_page)

            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            for i in range(len(resp)):
                URLs.append(resp[i]["git_url"])

            if include_gists is True:
                URLs.extend(self.UserGists(user, username=username, token=token))
            current_page += 1
        return(URLs)

    def fromOrg(self, org_name, username=None, token=None):
        """
        Retrieves a list of repositories for a Github organization.
        Input:-
        org_name: Github organization name.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/orgs/{0}/repos?per_page=40000000&page={1}".format(org_name, current_page)
            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            for i in range(len(resp)):
                URLs.append(resp[i]["git_url"])
            current_page += 1
        return(URLs)

    def fromOrgIncludeUsers(self, org_name, username=None, token=None, include_gists=False):
        """
        Retrieves a list of repositories for a Github organization
        and repositories of the Github organization's members.
        Input:-
        org_name: Github organization name.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """

        URLs = []
        members = []
        resp = []
        current_page = 1
        URLs.extend(self.fromOrg(org_name, username=username, token=token))

        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/orgs/{0}/members?per_page=40000000&page={1}".format(org_name, current_page)
            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            current_page += 1
            for i in range(len(resp)):
                members.append(resp[i]["login"])

        for member in members:
            URLs.extend(self.fromUser(member, username=username, token=token, include_gists=include_gists))

        return(URLs)

    def checkAuthentication(self, username, token):
        """
        Checks whether an authentication credentials are valid or not.
        Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        True: if the authentication credentials are valid.
        False: if the authentication credentials are invalid.
        """

        API = "https://api.github.com/user"
        resp = requests.get(API, auth=(username, token), timeout=self.timeout, headers=self.headers)
        if resp.status_code == 200:
            return(True)
        else:
            return(False)

    def checkResponse(self, response):
        """
        Validates whether there an error in the response.
        """
        try:
            if "API rate limit exceeded" in response["message"]:
                print('[!] Error: Github API rate limit exceeded')
                return(1)
        except TypeError:
            pass

        try:
            if (response["message"] == "Not Found"):
                return(2)  # The organization does not exist
        except TypeError:
            pass

        return(0)

    def fromAuthenticatedUser(self, username, token):
        """
        Retrieves a list of Github repositories than an authenticated user
        has access to.
        Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """
        URLs = []
        resp = []
        current_page = 1

        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/user/repos?per_page=40000000&type=all&page={}".format(current_page)
            resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            for i in range(len(resp)):
                URLs.append(resp[i]["git_url"])
            current_page += 1
        return(URLs)

def cloneRepo(URL, cloningpath, username=None, token=None):
    """
    Clones a single GIT repository.
    Input:-
    URL: GIT repository URL.
    cloningPath: the directory that the repository will be cloned at.
    Optional Input:-
    username: Github username.
    token: Github token or password.
    """

    try:
        try:
            if not os.path.exists(cloningpath):
                os.mkdir(cloningpath)
        except Exception:
            pass
        URL = URL.replace("git://", "https://")
        if (username or token) is not None:
            URL = URL.replace("https://", "https://{}:{}@".format(username, token))
        repopath = URL.split("/")[-2] + "_" + URL.split("/")[-1]
        if repopath.endswith(".git"):
            repopath = repopath[:-4]
        if '@' in repopath:
            repopath = repopath.replace(repopath[:repopath.index("@") + 1], "")
        fullpath = cloningpath + "/" + repopath
        with threading.Lock():
            print(fullpath)

        if os.path.exists(fullpath):
            git.Repo(fullpath).remote().pull()
        else:
            git.Repo.clone_from(URL, fullpath)
    except Exception:
        print("Error: There was an error in cloning [{}]".format(URL))

def cloneBulkRepos(URLs, cloningPath, threads_limit=5, username=None, token=None):
    """
    Clones a bulk of GIT repositories.
    Input:-
    URLs: A list of GIT repository URLs.
    cloningPath: the directory that the repository will be cloned at.
    Optional Input:-
    threads_limit: The limit of working threads.
    username: Github username.
    token: Github token or password.
    """

    Q = queue.Queue()
    threads_state = []
    for URL in URLs:
        Q.put(URL)
    while Q.empty() is False:
        if (threading.active_count() < (threads_limit + 1)):
            t = threading.Thread(target=cloneRepo, args=(Q.get(), cloningPath,), kwargs={"username": username, "token": token})
            t.daemon = True
            t.start()
        else:
            time.sleep(0.5)

            threads_state.append(t)
    for _ in threads_state:
        _.join()

def main():
    """
    The main function.
    """

    parser = argparse.ArgumentParser()
    parser.add_argument("-u", "--user",
                        dest="users",
                        help="Github user (comma-separated input for multiple Github users).",
                        action='store')
    parser.add_argument("-org", "--org",
                        dest="organizations",
                        help="Github organization (comma-separated input for multiple Github organizations).",
                        action='store')
    parser.add_argument("--include-org-members",
                        dest="include_organization_members",
                        help="Include the members of a Github organization.",
                        action='store_true')
    parser.add_argument("-o", "--output-path",
                        dest="output_path",
                        help="The directory to use in cloning Git repositories.",
                        action='store')
    parser.add_argument("-t", "--threads",
                        dest="threads_limit",
                        help="Threads used in cloning repositories (Default: 5).",
                        action='store',
                        default=5)
    parser.add_argument("-a", "--authentication",
                        dest="authentication",
                        help="Github authentication credentials (username:token).",
                        action='store')
    parser.add_argument("--include-authenticated-repos",
                        dest="include_authenticated_repos",
                        help="Include repositories that the authenticated Github" +
                             " account have access to.",
                        action='store_true')
    parser.add_argument("--include-gists",
                        dest="include_gists",
                        help="Include gists.",
                        action='store_true')
    parser.add_argument("--echo-urls",
                        dest="echo_urls",
                        help="Print gathered URLs only and then exit.",
                        action='store_true')
    args = parser.parse_args()

    users = args.users if args.users else None
    organizations = args.organizations if args.organizations else None
    include_organization_members = args.include_organization_members if args.include_organization_members else False
    output_path = args.output_path if args.output_path else None
    threads_limit = int(args.threads_limit) if args.threads_limit else 5
    authentication = args.authentication if args.authentication else None
    include_authenticated_repos = args.include_authenticated_repos if args.include_authenticated_repos else False
    include_gists = args.include_gists if args.include_gists else False
    echo_urls = args.echo_urls if args.echo_urls else False

    if threads_limit > 10:
        print("Error: Using more than 10 threads may cause errors.\nDecrease the amount of used threads.")
        print("\nExiting....")
        exit(1)

    if (not args.output_path) and (not echo_urls):
        print("Error: The output path is not specified.")
        print("\nExiting...")
        exit(1)

    if not (args.users or args.organizations):
        print("Error: Both Github users and Github organizations are not specified.")
        print("\nExiting...")
        exit(1)

    if str.isdigit(str(threads_limit)) is False:
        print("Error: Specified threads specified is invalid.")
        print("\nExiting...")
        exit(1)

    if not echo_urls:
        if not os.path.exists(output_path):
            os.mkdir(output_path)

    if authentication is not None:
        if ':' not in authentication:
            print('[!] Error: Incorrect authentication value, must be: <username>:<password_or_personal_access_token>')
            print('\nExiting...')
            exit(1)
        if getReposURLs().checkAuthentication(authentication.split(":")[0], authentication.split(":")[1]) is False:
            print("Error: authentication failed.")
            print("\nExiting...")
            exit(1)
        else:
            username = authentication.split(":")[0]
            token = authentication.split(":")[1]
    else:
        username = None
        token = None

    if (include_authenticated_repos is True) and (authentication is None):
        print("Error: --include-authenticated-repos is used and --authentication is not provided.")
        print("\nExiting...")
        exit(1)

    URLs = []

    if include_authenticated_repos is True:
        URLs.extend(getReposURLs().fromAuthenticatedUser(username, token))
        if include_gists is True:
            URLs.extend(getReposURLs().AuthenticatedGists(username, token))

    if users is not None:
        users = users.replace(" ", "").split(",")
        for user in users:
            URLs.extend(getReposURLs().fromUser(user, username=username, token=token, include_gists=include_gists))

    if organizations is not None:
        organizations = organizations.replace(" ", "").split(",")

        for organization in organizations:
            if include_organization_members is False:
                URLs.extend(getReposURLs().fromOrg(organization, username=username, token=token))
            else:
                URLs.extend(getReposURLs().fromOrgIncludeUsers(organization, username=username, token=token, include_gists=include_gists))

    URLs = list(set(URLs))
    if echo_urls is True:
        for _ in URLs:
            print(_)
        return

    cloneBulkRepos(URLs, output_path, threads_limit=threads_limit, username=username, token=token)

if (__name__ == "__main__"):
    try:
        main()
    except KeyboardInterrupt:
            print('\nKeyboardInterrupt Detected.')
            print('\nExiting...')
            exit(0)

# *** END *** #
<commit_msg>Added a check for queue module to fix a Python2 support issue.<commit_after>#!/usr/bin/env python3
# coding=utf-8
# *******************************************************************
# *** GithubCloner ***
# * Description:
#   A script that clones public Github repositories
#   of users and organizations automatically.
# * Version:
#   v0.3
# * Homepage:
#   https://github.com/mazen160/GithubCloner
# * Author:
#   Mazin Ahmed <Mazin AT MazinAhmed DOT net>
# *******************************************************************

# Modules
import argparse
import git
import json
import os
try:
    import queue
except ImportError:
    import Queue as queue
import requests
import threading
import time

class getReposURLs(object):
    def __init__(self):
        self.user_agent = "GithubCloner (https://github.com/mazen160/GithubCloner)"
        self.headers = {'User-Agent': self.user_agent, 'Accept': '*/*'}
        self.timeout = 3

    def UserGists(self, user, username=None, token=None):
        """
        Returns a list of GIT URLs for accessible gists.
        Input:-
        user: Github user.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github gist repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/users/{0}/gists?page={1}".format(user, current_page)
            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            for i in range(len(resp)):
                URLs.append(resp[i]["git_pull_url"])
            current_page += 1
        return(URLs)

    def AuthenticatedGists(self, username, token):
        """
        Returns a list of gists of an authenticated user.
        Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github gist repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/gists?page={0}".format(current_page)
            resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)
            for i in range(len(resp)):
                URLs.append(resp[i]["git_pull_url"])
            current_page += 1

        return(URLs)

    def fromUser(self, user, username=None, token=None, include_gists=False):
        """
        Retrieves a list of repositories for a Github user.
        Input:-
        user: Github username.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/users/{0}/repos?per_page=40000000&page={1}".format(user, current_page)

            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            for i in range(len(resp)):
                URLs.append(resp[i]["git_url"])

            if include_gists is True:
                URLs.extend(self.UserGists(user, username=username, token=token))
            current_page += 1
        return(URLs)

    def fromOrg(self, org_name, username=None, token=None):
        """
        Retrieves a list of repositories for a Github organization.
        Input:-
        org_name: Github organization name.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """

        URLs = []
        resp = []
        current_page = 1
        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/orgs/{0}/repos?per_page=40000000&page={1}".format(org_name, current_page)
            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            for i in range(len(resp)):
                URLs.append(resp[i]["git_url"])
            current_page += 1
        return(URLs)

    def fromOrgIncludeUsers(self, org_name, username=None, token=None, include_gists=False):
        """
        Retrieves a list of repositories for a Github organization
        and repositories of the Github organization's members.
        Input:-
        org_name: Github organization name.
        Optional Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """

        URLs = []
        members = []
        resp = []
        current_page = 1
        URLs.extend(self.fromOrg(org_name, username=username, token=token))

        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/orgs/{0}/members?per_page=40000000&page={1}".format(org_name, current_page)
            if (username or token) is None:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout).text
            else:
                resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            if self.checkResponse(resp) != 0:
                return([])

            current_page += 1
            for i in range(len(resp)):
                members.append(resp[i]["login"])

        for member in members:
            URLs.extend(self.fromUser(member, username=username, token=token, include_gists=include_gists))

        return(URLs)

    def checkAuthentication(self, username, token):
        """
        Checks whether an authentication credentials are valid or not.
        Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        True: if the authentication credentials are valid.
        False: if the authentication credentials are invalid.
        """

        API = "https://api.github.com/user"
        resp = requests.get(API, auth=(username, token), timeout=self.timeout, headers=self.headers)
        if resp.status_code == 200:
            return(True)
        else:
            return(False)

    def checkResponse(self, response):
        """
        Validates whether there an error in the response.
        """
        try:
            if "API rate limit exceeded" in response["message"]:
                print('[!] Error: Github API rate limit exceeded')
                return(1)
        except TypeError:
            pass

        try:
            if (response["message"] == "Not Found"):
                return(2)  # The organization does not exist
        except TypeError:
            pass

        return(0)

    def fromAuthenticatedUser(self, username, token):
        """
        Retrieves a list of Github repositories than an authenticated user
        has access to.
        Input:-
        username: Github username.
        token: Github token or password.
        Output:-
        a list of Github repositories URLs.
        """
        URLs = []
        resp = []
        current_page = 1

        while (len(resp) != 0 or current_page == 1):
            API = "https://api.github.com/user/repos?per_page=40000000&type=all&page={}".format(current_page)
            resp = requests.get(API, headers=self.headers, timeout=self.timeout, auth=(username, token)).text
            resp = json.loads(resp)

            for i in range(len(resp)):
                URLs.append(resp[i]["git_url"])
            current_page += 1
        return(URLs)

def cloneRepo(URL, cloningpath, username=None, token=None):
    """
    Clones a single GIT repository.
    Input:-
    URL: GIT repository URL.
    cloningPath: the directory that the repository will be cloned at.
    Optional Input:-
    username: Github username.
    token: Github token or password.
    """

    try:
        try:
            if not os.path.exists(cloningpath):
                os.mkdir(cloningpath)
        except Exception:
            pass
        URL = URL.replace("git://", "https://")
        if (username or token) is not None:
            URL = URL.replace("https://", "https://{}:{}@".format(username, token))
        repopath = URL.split("/")[-2] + "_" + URL.split("/")[-1]
        if repopath.endswith(".git"):
            repopath = repopath[:-4]
        if '@' in repopath:
            repopath = repopath.replace(repopath[:repopath.index("@") + 1], "")
        fullpath = cloningpath + "/" + repopath
        with threading.Lock():
            print(fullpath)

        if os.path.exists(fullpath):
            git.Repo(fullpath).remote().pull()
        else:
            git.Repo.clone_from(URL, fullpath)
    except Exception:
        print("Error: There was an error in cloning [{}]".format(URL))

def cloneBulkRepos(URLs, cloningPath, threads_limit=5, username=None, token=None):
    """
    Clones a bulk of GIT repositories.
    Input:-
    URLs: A list of GIT repository URLs.
    cloningPath: the directory that the repository will be cloned at.
    Optional Input:-
    threads_limit: The limit of working threads.
    username: Github username.
    token: Github token or password.
    """

    Q = queue.Queue()
    threads_state = []
    for URL in URLs:
        Q.put(URL)
    while Q.empty() is False:
        if (threading.active_count() < (threads_limit + 1)):
            t = threading.Thread(target=cloneRepo, args=(Q.get(), cloningPath,), kwargs={"username": username, "token": token})
            t.daemon = True
            t.start()
        else:
            time.sleep(0.5)

            threads_state.append(t)
    for _ in threads_state:
        _.join()

def main():
    """
    The main function.
    """

    parser = argparse.ArgumentParser()
    parser.add_argument("-u", "--user",
                        dest="users",
                        help="Github user (comma-separated input for multiple Github users).",
                        action='store')
    parser.add_argument("-org", "--org",
                        dest="organizations",
                        help="Github organization (comma-separated input for multiple Github organizations).",
                        action='store')
    parser.add_argument("--include-org-members",
                        dest="include_organization_members",
                        help="Include the members of a Github organization.",
                        action='store_true')
    parser.add_argument("-o", "--output-path",
                        dest="output_path",
                        help="The directory to use in cloning Git repositories.",
                        action='store')
    parser.add_argument("-t", "--threads",
                        dest="threads_limit",
                        help="Threads used in cloning repositories (Default: 5).",
                        action='store',
                        default=5)
    parser.add_argument("-a", "--authentication",
                        dest="authentication",
                        help="Github authentication credentials (username:token).",
                        action='store')
    parser.add_argument("--include-authenticated-repos",
                        dest="include_authenticated_repos",
                        help="Include repositories that the authenticated Github" +
                             " account have access to.",
                        action='store_true')
    parser.add_argument("--include-gists",
                        dest="include_gists",
                        help="Include gists.",
                        action='store_true')
    parser.add_argument("--echo-urls",
                        dest="echo_urls",
                        help="Print gathered URLs only and then exit.",
                        action='store_true')
    args = parser.parse_args()

    users = args.users if args.users else None
    organizations = args.organizations if args.organizations else None
    include_organization_members = args.include_organization_members if args.include_organization_members else False
    output_path = args.output_path if args.output_path else None
    threads_limit = int(args.threads_limit) if args.threads_limit else 5
    authentication = args.authentication if args.authentication else None
    include_authenticated_repos = args.include_authenticated_repos if args.include_authenticated_repos else False
    include_gists = args.include_gists if args.include_gists else False
    echo_urls = args.echo_urls if args.echo_urls else False

    if threads_limit > 10:
        print("Error: Using more than 10 threads may cause errors.\nDecrease the amount of used threads.")
        print("\nExiting....")
        exit(1)

    if (not args.output_path) and (not echo_urls):
        print("Error: The output path is not specified.")
        print("\nExiting...")
        exit(1)

    if not (args.users or args.organizations):
        print("Error: Both Github users and Github organizations are not specified.")
        print("\nExiting...")
        exit(1)

    if str.isdigit(str(threads_limit)) is False:
        print("Error: Specified threads specified is invalid.")
        print("\nExiting...")
        exit(1)

    if not echo_urls:
        if not os.path.exists(output_path):
            os.mkdir(output_path)

    if authentication is not None:
        if ':' not in authentication:
            print('[!] Error: Incorrect authentication value, must be: <username>:<password_or_personal_access_token>')
            print('\nExiting...')
            exit(1)
        if getReposURLs().checkAuthentication(authentication.split(":")[0], authentication.split(":")[1]) is False:
            print("Error: authentication failed.")
            print("\nExiting...")
            exit(1)
        else:
            username = authentication.split(":")[0]
            token = authentication.split(":")[1]
    else:
        username = None
        token = None

    if (include_authenticated_repos is True) and (authentication is None):
        print("Error: --include-authenticated-repos is used and --authentication is not provided.")
        print("\nExiting...")
        exit(1)

    URLs = []

    if include_authenticated_repos is True:
        URLs.extend(getReposURLs().fromAuthenticatedUser(username, token))
        if include_gists is True:
            URLs.extend(getReposURLs().AuthenticatedGists(username, token))

    if users is not None:
        users = users.replace(" ", "").split(",")
        for user in users:
            URLs.extend(getReposURLs().fromUser(user, username=username, token=token, include_gists=include_gists))

    if organizations is not None:
        organizations = organizations.replace(" ", "").split(",")

        for organization in organizations:
            if include_organization_members is False:
                URLs.extend(getReposURLs().fromOrg(organization, username=username, token=token))
            else:
                URLs.extend(getReposURLs().fromOrgIncludeUsers(organization, username=username, token=token, include_gists=include_gists))

    URLs = list(set(URLs))
    if echo_urls is True:
        for _ in URLs:
            print(_)
        return

    cloneBulkRepos(URLs, output_path, threads_limit=threads_limit, username=username, token=token)

if (__name__ == "__main__"):
    try:
        main()
    except KeyboardInterrupt:
            print('\nKeyboardInterrupt Detected.')
            print('\nExiting...')
            exit(0)

# *** END *** #
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2011-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
# Released under the terms of the MIT License. See LICENSE for details.

from __future__ import print_function

import os

from colorama import Fore, Style

from .migrate import run_migrations

__all__ = ["get_default_config_path", "get_bookmarks", "add_bookmarks",
           "delete_bookmarks", "list_bookmarks"]

YELLOW = Fore.YELLOW + Style.BRIGHT
RED = Fore.RED + Style.BRIGHT

INDENT1 = " " * 3

def _ensure_dirs(path):
    """Ensure the directories within the given pathname exist."""
    dirname = os.path.dirname(path)
    if dirname and not os.path.exists(dirname):  # Race condition, meh...
        os.makedirs(dirname)

def _load_config_file(config_path=None):
    """Read the config file and return a config parser object."""
    run_migrations()
    cfg_path = config_path or get_default_config_path()

    try:
        with open(cfg_path, "rb") as config_file:
            return config_file.read().split("\n")
    except IOError:
        return []

def _save_config_file(config, config_path=None):
    """Save config changes to the given config file."""
    run_migrations()
    cfg_path = config_path or get_default_config_path()
    _ensure_dirs(cfg_path)

    with open(cfg_path, "wb") as config_file:
        config_file.write("\n".join(config))

def get_default_config_path():
    """Return the default path to the configuration file."""
    xdg_cfg = os.environ.get("XDG_CONFIG_HOME") or os.path.join("~", ".config")
    return os.path.join(os.path.expanduser(xdg_cfg), "gitup", "bookmarks")

def get_bookmarks(config_path=None):
    """Get a list of all bookmarks, or an empty list if there are none."""
    return _load_config_file(config_path)

def add_bookmarks(paths, config_path=None):
    """Add a list of paths as bookmarks to the config file."""
    config = _load_config_file(config_path)
    added, exists = [], []
    for path in paths:
        path = os.path.abspath(path)
        if path in config:
            exists.append(path)
        else:
            config.append(path)
            added.append(path)
    _save_config_file(config, config_path)

    if added:
        print(YELLOW + "Added bookmarks:")
        for path in added:
            print(INDENT1, path)
    if exists:
        print(RED + "Already bookmarked:")
        for path in exists:
            print(INDENT1, path)

def delete_bookmarks(paths, config_path=None):
    """Remove a list of paths from the bookmark config file."""
    config = _load_config_file(config_path)

    deleted, notmarked = [], []
    if config:
        for path in paths:
            path = os.path.abspath(path)
            if path in config:
                config.remove(path)
                deleted.append(path)
            else:
                notmarked.append(path)
        _save_config_file(config, config_path)
    else:
        notmarked = [os.path.abspath(path) for path in paths]

    if deleted:
        print(YELLOW + "Deleted bookmarks:")
        for path in deleted:
            print(INDENT1, path)
    if notmarked:
        print(RED + "Not bookmarked:")
        for path in notmarked:
            print(INDENT1, path)

def list_bookmarks(config_path=None):
    """Print all of our current bookmarks."""
    bookmarks = get_bookmarks(config_path=config_path)
    if bookmarks:
        print(YELLOW + "Current bookmarks:")
        for bookmark_path in bookmarks:
            print(INDENT1, bookmark_path)
    else:
        print("You have no bookmarks to display.")
<commit_msg>Fix bookmarks on Python 3.<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2011-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
# Released under the terms of the MIT License. See LICENSE for details.

from __future__ import print_function

import os

from colorama import Fore, Style

from .migrate import run_migrations

__all__ = ["get_default_config_path", "get_bookmarks", "add_bookmarks",
           "delete_bookmarks", "list_bookmarks"]

YELLOW = Fore.YELLOW + Style.BRIGHT
RED = Fore.RED + Style.BRIGHT

INDENT1 = " " * 3

def _ensure_dirs(path):
    """Ensure the directories within the given pathname exist."""
    dirname = os.path.dirname(path)
    if dirname and not os.path.exists(dirname):  # Race condition, meh...
        os.makedirs(dirname)

def _load_config_file(config_path=None):
    """Read the config file and return a list of bookmarks."""
    run_migrations()
    cfg_path = config_path or get_default_config_path()

    try:
        with open(cfg_path, "r") as config_file:
            return config_file.read().split("\n")
    except IOError:
        return []

def _save_config_file(config, config_path=None):
    """Save config changes to the given config file."""
    run_migrations()
    cfg_path = config_path or get_default_config_path()
    _ensure_dirs(cfg_path)

    with open(cfg_path, "w") as config_file:
        config_file.write("\n".join(config))

def get_default_config_path():
    """Return the default path to the configuration file."""
    xdg_cfg = os.environ.get("XDG_CONFIG_HOME") or os.path.join("~", ".config")
    return os.path.join(os.path.expanduser(xdg_cfg), "gitup", "bookmarks")

def get_bookmarks(config_path=None):
    """Get a list of all bookmarks, or an empty list if there are none."""
    return _load_config_file(config_path)

def add_bookmarks(paths, config_path=None):
    """Add a list of paths as bookmarks to the config file."""
    config = _load_config_file(config_path)
    added, exists = [], []
    for path in paths:
        path = os.path.abspath(path)
        if path in config:
            exists.append(path)
        else:
            config.append(path)
            added.append(path)
    _save_config_file(config, config_path)

    if added:
        print(YELLOW + "Added bookmarks:")
        for path in added:
            print(INDENT1, path)
    if exists:
        print(RED + "Already bookmarked:")
        for path in exists:
            print(INDENT1, path)

def delete_bookmarks(paths, config_path=None):
    """Remove a list of paths from the bookmark config file."""
    config = _load_config_file(config_path)

    deleted, notmarked = [], []
    if config:
        for path in paths:
            path = os.path.abspath(path)
            if path in config:
                config.remove(path)
                deleted.append(path)
            else:
                notmarked.append(path)
        _save_config_file(config, config_path)
    else:
        notmarked = [os.path.abspath(path) for path in paths]

    if deleted:
        print(YELLOW + "Deleted bookmarks:")
        for path in deleted:
            print(INDENT1, path)
    if notmarked:
        print(RED + "Not bookmarked:")
        for path in notmarked:
            print(INDENT1, path)

def list_bookmarks(config_path=None):
    """Print all of our current bookmarks."""
    bookmarks = get_bookmarks(config_path=config_path)
    if bookmarks:
        print(YELLOW + "Current bookmarks:")
        for bookmark_path in bookmarks:
            print(INDENT1, bookmark_path)
    else:
        print("You have no bookmarks to display.")
<|endoftext|>