# HG changeset patch
# User Jeff Hammel <k0scist@gmail.com>
# Date 1486352408 0
# Node ID 6873638f82cee7a210ca28b7612d984251ab6f35

initial commit

diff -r 000000000000 -r 6873638f82ce README.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/README.txt	Mon Feb 06 03:40:08 2017 +0000
@@ -0,0 +1,61 @@
+# pypedream
+
+python + other packaging works
+
+      ______
+     |______|
+      |    |
+      |____|
+
+
+
+## What's inside
+
+This is a toolbox for dealing with packages as it stands.  Install
+this your usual way, e.g.:  `python setup.py develop`
+
+## Console scripts
+
+Invoke with `--help` for usage info.
+
+- `py2rpm`: script for fetching an python package from http://pypi.python.org/
+  and its dependencies using `pip` and converting to RPMs using
+  `python setup.py bdist_rpm`.  Requires the `pip` and `rpmbuild` on
+  your system.
+
+- `py2centos`: the problem with using `bdist_rpm` is that it converts
+  the python package to an RPM as appropriate for your current
+  environment.  What is desired is a package for, at current, the
+  Centos 6 operating system.  In order to surmount these difficulties,
+  Docker is used by `py2centos` to provision a Centos 6 container,
+  copy the `py2rpm.py` python script into the container, and set up an
+  execution endpoint that will generate the Centos RPMs in an output
+  directory.
+
+
+## fpm
+
+`fpm` is a ruby package manipulation tool.  You can use `py2rpm` with
+the `--fpm` option, if you have the `fpm` command line program on your
+path, to do the conversion with this tool.  The advantage is that the
+package dependencies are noted correctly.  By default, an additional
+namespace of `python-` is prepended to the package name.  This is
+probably desirable.
+
+
+## RPM
+
+Querying rpm dependencies:
+
+    # rpm -qpR *.rpm
+
+Listing files in an rpm:
+
+    # rpm -qlp *.rpm
+
+
+## Links
+
+- https://github.com/jordansissel/fpm
+- https://github.com/jordansissel/fpm/wiki/ConvertingPython
+- http://www.alexhudson.com/2013/05/24/packaging-a-virtualenv-really-not-relocatable/
diff -r 000000000000 -r 6873638f82ce pypedream/Dockerfile
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pypedream/Dockerfile	Mon Feb 06 03:40:08 2017 +0000
@@ -0,0 +1,60 @@
+# a Docker image for building RPMs native to centos from python packages
+# This uses fpm:
+# https://github.com/jordansissel/fpm
+
+# image
+# https://hub.docker.com/_/centos/
+FROM centos:{{ version }}
+
+# environment
+ENV working_dir /opt/env
+WORKDIR $working_dir
+RUN mkdir -p $working_dir
+
+# set up yum repo
+RUN yum clean all
+RUN yum -y update
+
+# install EPEL:
+# Alternative RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+RUN yum -y install epel-release
+
+# install dependencies
+RUN yum -y install ruby-devel gcc rubygems tar rpm-build which
+
+# Our ruby is too old!
+# Step 9 : RUN gem install fpm
+#  ---> Running in c2fc672985ea
+# ERROR:  Error installing fpm:
+# 	ruby-xz requires Ruby version >= 1.9.3.
+# So
+#    https://gist.github.com/slouma2000/8619039
+# and
+#    http://stackoverflow.com/questions/28129438/docker-rvm-command-not-found
+RUN     gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+RUN curl -L get.rvm.io | bash -s stable
+RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm install 1.9.3"
+RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm use 1.9.3 --default"
+
+# Update rubygems
+RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && gem update --system"
+RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && gem install bundler"
+
+# install fpm
+RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && gem install fpm"
+
+# install pip + dependencies
+RUN yum -y install python-pip python-argparse
+RUN pip install --upgrade pip
+
+# COPY script
+# https://docs.docker.com/engine/reference/builder/#copy
+COPY py2rpm.py $working_dir
+
+# ENTRYPOINT to generate RPMs
+# https://docs.docker.com/engine/reference/builder/#entrypoint
+# Invoke like:
+# (lcars)# docker run 3e4251a30f93 m3u8
+# or
+# docker run -v  /Users/jehammel/foo/:/opt/out 3e4251a30f93 m3u8 -o /opt/out
+ENTRYPOINT ["python", "py2rpm.py"]
diff -r 000000000000 -r 6873638f82ce pypedream/__init__.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pypedream/__init__.py	Mon Feb 06 03:40:08 2017 +0000
@@ -0,0 +1,2 @@
+#
+
diff -r 000000000000 -r 6873638f82ce pypedream/eval.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pypedream/eval.py	Mon Feb 06 03:40:08 2017 +0000
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+evaluate a bash script or command
+"""
+
+# imports
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+
+
+def parse_env(output):
+    """parse environment content a la `env` output"""
+
+    output = output.strip()
+    lines = output.splitlines()
+    return dict([line.split('=', 1) for line in lines])
+
+
+def source(src, update=True):
+    """
+    eval a file and return the environment dictionary
+    """
+    command = 'set -e && source "{src}" && env'.format(src=src)
+    output = subprocess.check_output(command, shell=True)
+    env = parse_env(output)
+    if update:
+        os.environ.update(env)
+    return env
+
+
+def eval_command(command, update=True):
+    """
+    evaluate a command
+
+    command -- command to evaluate
+    update -- whether to update os.environ with result
+    """
+
+    output = subprocess.check_output(command, shell=True)
+    fd, name = tempfile.mkstemp()
+    try:
+        os.write(fd, output)
+        os.close(fd)
+        env = source(name, update=update)
+    finally:
+        try:
+            os.remove(name)
+        except:
+            pass
+    return env
+
+
+def main(args=sys.argv[1:]):
+
+    # parse command line
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('command', help="command to run")
+    options = parser.parse_args(args)
+
+    # keep copy of the old env
+    old_env = os.environ.copy()
+
+    # calcaulate the new environment
+    new_env = eval_command(options.command)
+
+    # compute difference
+    updated = []
+    for key, value in new_env.items():
+        if value != old_env.get(key):
+            updated.append(key)
+    updated.sort()
+
+    # output
+    for key in updated:
+        print ('{key}={value}'.format(key=key, value=new_env[key]))
+
+
+if __name__ == '__main__':
+    main()
diff -r 000000000000 -r 6873638f82ce pypedream/py2centos.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pypedream/py2centos.py	Mon Feb 06 03:40:08 2017 +0000
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+convert python packages to Centos(6) RPMs using Docker
+"""
+
+# imports
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from eval import eval_command
+from jinja2 import Template
+
+# module globals
+here = os.path.dirname(os.path.realpath(__file__))
+
+
+def ensure_dir(directory):
+    """ensure a directory exists"""
+
+    if os.path.exists(directory):
+        if not os.path.isdir(directory):
+            raise OSError("Not a directory: '{}'".format(directory))
+        return directory
+    os.makedirs(directory)
+    return directory
+
+
+def main(args=sys.argv[1:]):
+    """CLI"""
+
+    # parse command line
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('package')
+    parser.add_argument('-o', '--output-dir', dest='output_dir',
+                        type=ensure_dir, default='.',
+                        help="directory for output [DEFAULT: %(default)s]")
+    parser.add_argument('--fpm', '--use-fpm', dest='use_fpm',
+                        action='store_true', default=False,
+                        help="use fpm for package conversion")
+    parser.add_argument('--version', '--centos-version', dest='version',
+                        default='6',
+                        help="Centos version to build under [DEFAULT: %(default)s]")
+    parser.add_argument('-v', '--verbose', dest='verbose',
+                        action='store_true', default=False,
+                        help="enable verbose output")
+    options = parser.parse_args(args)
+
+    # load the Dockerfile template contents
+    dockerfile = os.path.join(here, 'Dockerfile')
+    with open(dockerfile, 'r') as f:
+        contents = f.read()
+    template = Template(contents)
+
+    tempdir = tempfile.mkdtemp()
+    try:
+        # output the rendered template
+        output = os.path.join(tempdir, 'Dockerfile')
+        data = {'version': options.version}
+        with open(output, 'w') as f:
+            f.write(template.render(**data))
+
+        # copy the script in so that it may be picked up by `docker build`
+        # see https://docs.python.org/2/library/shutil.html
+        script = 'py2rpm.py'
+        src = os.path.join(here, script)
+        shutil.copy2(src, tempdir)
+
+        # build the machine
+        # see also http://blog.endpoint.com/2015/01/getting-realtime-output-using-python.html
+        command = ['docker', 'build', '.']
+        output = subprocess.check_output(command, cwd=tempdir)
+        last_line = output.strip().splitlines()[-1].strip()
+        print (last_line)
+        if not last_line.startswith('Success'):
+            raise AssertionError(last_line)
+        container = last_line.split()[-1]
+        if options.verbose:
+            print ("Container: {container}".format(container=container))
+    except subprocess.CalledProcessError as e:
+        print ("Directory: {tempdir}".format(tempdir=tempdir))
+        print ("{command} exited with {returncode}".format(command=e.cmd,
+                                                           returncode=e.returncode))
+        print (e.output)
+        import pdb; pdb.set_trace()
+    finally:
+        shutil.rmtree(tempdir)
+
+    # run the container + mount the docker volume (and such)
+    # https://docs.docker.com/engine/userguide/containers/dockervolumes/#mount-a-host-directory-as-a-data-volume
+    # (lcars)# docker run -v  /Users/jehammel/foo/:/opt/out 3e4251a30f93 m3u8 -o /opt/out
+    output_dir = os.path.abspath(options.output_dir)
+    command = ['docker',
+               'run',
+               '-v',
+               '{0}:/opt/out'.format(output_dir),
+               container,
+               '-o',
+               '/opt/out',
+               options.package]
+    if options.use_fpm:
+        command.append('--use-fpm')
+    if options.verbose:
+        print ("Running:\n{command}".format(command=subprocess.list2cmdline(command)))
+    try:
+        subprocess.check_output(command)
+    except subprocess.CalledProcessError as e:
+        import pdb; pdb.set_trace()
+        print (subprocess.list2cmdline(command))
+        raise
+
+
+if __name__ == '__main__':
+    main()
diff -r 000000000000 -r 6873638f82ce pypedream/py2rpm.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pypedream/py2rpm.py	Mon Feb 06 03:40:08 2017 +0000
@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+
+"""
+convert a python package to RPM
+"""
+
+# Links:
+# - https://pypi.python.org/pypi/py2pack : generates spec files!  However, requires pypi as a source :(
+# - http://wheel.readthedocs.org/en/latest/ : untried! Not sure how it solves the problem
+# - https://docs.python.org/2.0/dist/creating-rpms.html : Not very helpful!
+# - https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.pip.html : Probably the way to go!
+#   * see also https://salt.readthedocs.org/en/v0.17.1/ref/states/all/salt.states.pip_state.html
+# - https://github.com/jordansissel/fpm : Does all the things!  However, it will use host not guest operating system :( and does not operate on files :(
+# - http://blog.ziade.org/2011/03/25/bdist_rpm-is-dead-long-life-to-py2rpm/ : doesn't actually find any files!
+# - https://github.com/fedora-python/pyp2rpm : Doesn't seem to work!
+
+# imports
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def ensure_dir(directory):
+    """ensure a directory exists"""
+
+    if os.path.exists(directory):
+        if not os.path.isdir(directory):
+            raise OSError("Not a directory: '{}'".format(directory))
+
+        return directory
+
+    os.makedirs(directory)
+    return directory
+
+
+def filename2data(filename):
+    ext = '.tar.gz'
+    if not filename.endswith(ext):
+        raise NotImplementedError("""Oops!  I didn't think you would see this!
+I didn't have time to figure out how python packaging actually resolves
+        names of python packages, and for that, I apologize.  So I only support
+one format: .tar.gz""")
+    return filename.rsplit(ext)[0].split('-', 1)
+
+
+def directory2packagedata(directory):
+    """expects directory to ONLY have packages in it"""
+
+    retval = []
+    for package in os.listdir(directory):
+        name, version = filename2data(package)
+        retval.append(dict(name=name,
+                           version=version,
+                           basename=package,
+                           path=os.path.join(directory, package)))
+    return dict([(item['name'], item)
+                 for item in retval])
+
+
+def untar(tarball, destination):
+    """untar a `tarball` to `destination`"""
+    # TODO: this can be done without shelling out
+
+    command = ['tar', 'xzf', tarball, '-C', destination]
+    subprocess.check_call(command)
+
+
+def bdist_rpm(directory):
+    """build a directory with a setup.py and return the RPM"""
+
+    command = [sys.executable, 'setup.py', 'bdist_rpm']
+    subprocess.check_call(command, cwd=directory)
+    dist = os.path.join(directory, 'dist')
+    contents = os.listdir(dist)
+    ext = '.noarch.rpm'
+    rpms = [f for f in contents
+            if f.endswith(ext)]
+    if len(rpms) != 1:
+        raise AssertionError("Expecting one RPM file, found {0}".format(len(rpms)))
+    return os.path.abspath(os.path.join(dist, rpms.pop()))
+
+
+def fpm_converter(directory):
+    """
+    use `fpm` within a directory and return the RPM
+    See:
+    https://github.com/jordansissel/fpm/wiki/ConvertingPython
+    """
+    ext = '.rpm'
+    contents = os.listdir(directory)
+    assert 'setup.py' in contents
+    assert not bool([i for i in contents if i.endswith(ext)])
+    command = ['fpm', '--rpm-os', 'linux', '-s', 'python', '-t', 'rpm', 'setup.py']
+    subprocess.check_call(command, cwd=directory)
+    contents = os.listdir(directory)
+    rpms = [i for i in contents if i.endswith(ext)]
+    if len(rpms) != 1:
+        raise AssertionError("Expecting one RPM file, found {0}".format(len(rpms)))
+    return os.path.abspath(os.path.join(directory, rpms.pop()))
+
+
+def main(args=sys.argv[1:]):
+    """CLI"""
+
+    # parse command line
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('package')
+    parser.add_argument('-o', '--output-dir', dest='output_dir',
+                        type=ensure_dir, default='.',
+                        help="directory for output [DEFAULT: %(default)s]")
+    parser.add_argument('--fpm', '--use-fpm', dest='use_fpm',
+                        action='store_true', default=False,
+                        help="use fpm for package conversion")
+    parser.add_argument('-v', '--verbose', dest='verbose',
+                        action='store_true', default=False,
+                        help="be verbose")
+    options = parser.parse_args(args)
+
+    # make a temporary directory
+    tmpdir = tempfile.mkdtemp()
+    try:
+
+        # `pip` command to download the package:
+        # http://stackoverflow.com/questions/7300321/how-to-use-pythons-pip-to-download-and-keep-the-zipped-files-for-a-package
+        command = ['pip', 'install',
+                   '--download', tmpdir,
+                   options.package,
+                   '--no-use-wheel']
+        try:
+            subprocess.check_call(command)
+        except Exception as e:
+            print subprocess.list2cmdline(command)
+            raise
+
+        # get the package data
+        packagedata = directory2packagedata(tmpdir)
+        if options.verbose:
+            print (json.dumps(packagedata, sort_keys=True, indent=2))
+
+        # ensure we have the one package we specified;
+        # the rest are dependencies
+        if options.package not in packagedata:
+            raise AssertionError("Package '{}' not in packagedata: {}".format(options.package, json.dumps(packagedata)))
+
+        # build each package
+        for package, metadata in packagedata.items():
+            path = metadata['path']
+            print (package)
+            print (path)
+
+            # make a `build` directory
+            builddir = tempfile.mkdtemp()
+            try:
+                # inflate the tarball
+                untar(path, builddir)
+                contents = os.listdir(builddir)
+                print (",".join(contents))
+                if len(contents) != 1:
+                    raise AssertionError("Ooops!")
+
+                # find the package directory
+                subdir = os.path.join(builddir, contents.pop())
+                if not os.path.isdir(subdir):
+                    raise AssertionError("Ooops!")
+                contents = os.listdir(subdir)
+                print (",".join(contents))
+                if 'setup.py' not in contents:
+                    raise AssertionError("Cannot find setup.py")
+
+                # build the rpm
+                if options.use_fpm:
+                    rpm = fpm_converter(subdir)
+                else:
+                    rpm = bdist_rpm(subdir)
+
+                # move it to output directory
+                shutil.move(rpm, options.output_dir)
+
+            finally:
+                # cleanup
+                if True:
+                    shutil.rmtree(builddir, ignore_errors=True)
+                else:
+                    print ("build directory: {}".format(builddir))
+
+    finally:
+        # cleanup
+        if True:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+        else:
+            # print tmpdir contents
+            print ("{} :".format(tmpdir))
+            print ('\n'.join(sorted(os.listdir(tmpdir))))
+
+if __name__ == '__main__':
+    main()
diff -r 000000000000 -r 6873638f82ce setup.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/setup.py	Mon Feb 06 03:40:08 2017 +0000
@@ -0,0 +1,45 @@
+"""
+setup packaging script for pypedream
+"""
+
+import os
+
+version = "0.3"
+dependencies = ['PyYAML',
+                'Jinja2']
+
+# allow use of setuptools/distribute or distutils
+kw = {}
+try:
+    from setuptools import setup
+    kw['entry_points'] = """
+    [console_scripts]
+    py2rpm = pypedream.py2rpm:main
+    py2centos = pypedream.py2centos:main
+"""
+    kw['install_requires'] = dependencies
+except ImportError:
+    from distutils.core import setup
+    kw['requires'] = dependencies
+
+try:
+    here = os.path.dirname(os.path.abspath(__file__))
+    description = file(os.path.join(here, 'README.txt')).read()
+except IOError:
+    description = ''
+
+
+setup(name='pypedream',
+      version=version,
+      description="python packaging work",
+      long_description=description,
+      classifiers=[], # Get strings from http://www.python.org/pypi?%3Aaction=list_classifiers
+      author='Jeff Hammel',
+      author_email='k0scist@gmail.com',
+      url='',
+      license='',
+      packages=['pypedream'],
+      include_package_data=True,
+      zip_safe=False,
+      **kw
+      )