Mercurial > hg > pypedream
changeset 0:6873638f82ce default tip
initial commit
| author | Jeff Hammel <k0scist@gmail.com> |
|---|---|
| date | Mon, 06 Feb 2017 03:40:08 +0000 |
| parents | |
| children | |
| files | README.txt pypedream/Dockerfile pypedream/__init__.py pypedream/eval.py pypedream/py2centos.py pypedream/py2rpm.py setup.py |
| diffstat | 7 files changed, 571 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.txt Mon Feb 06 03:40:08 2017 +0000 @@ -0,0 +1,61 @@ +# pypedream + +python + other packaging works + + ______ + |______| + | | + |____| + + + +## What's inside + +This is a toolbox for dealing with packages as it stands. Install +this your usual way, e.g.: `python setup.py develop` + +## Console scripts + +Invoke with `--help` for usage info. + +- `py2rpm`: script for fetching an python package from http://pypi.python.org/ + and its dependencies using `pip` and converting to RPMs using + `python setup.py bdist_rpm`. Requires the `pip` and `rpmbuild` on + your system. + +- `py2centos`: the problem with using `bdist_rpm` is that it converts + the python package to an RPM as appropriate for your current + environment. What is desired is a package for, at current, the + Centos 6 operating system. In order to surmount these difficulties, + Docker is used by `py2centos` to provision a Centos 6 container, + copy the `py2rpm.py` python script into the container, and set up an + execution endpoint that will generate the Centos RPMs in an output + directory. + + +## fpm + +`fpm` is a ruby package manipulation tool. You can use `py2rpm` with +the `--fpm` option, if you have the `fpm` command line program on your +path, to do the conversion with this tool. The advantage is that the +package dependencies are noted correctly. By default, an additional +namespace of `python-` is prepended to the package name. This is +probably desirable. + + +## RPM + +Querying rpm dependencies: + + # rpm -qpR *.rpm + +Listing files in an rpm: + + # rpm -qlp *.rpm + + +## Links + +- https://github.com/jordansissel/fpm +- https://github.com/jordansissel/fpm/wiki/ConvertingPython +- http://www.alexhudson.com/2013/05/24/packaging-a-virtualenv-really-not-relocatable/
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pypedream/Dockerfile Mon Feb 06 03:40:08 2017 +0000 @@ -0,0 +1,60 @@ +# a Docker image for building RPMs native to centos from python packages +# This uses fpm: +# https://github.com/jordansissel/fpm + +# image +# https://hub.docker.com/_/centos/ +FROM centos:{{ version }} + +# environment +ENV working_dir /opt/env +WORKDIR $working_dir +RUN mkdir -p $working_dir + +# set up yum repo +RUN yum clean all +RUN yum -y update + +# install EPEL: +# Alternative RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm +RUN yum -y install epel-release + +# install dependencies +RUN yum -y install ruby-devel gcc rubygems tar rpm-build which + +# Our ruby is too old! +# Step 9 : RUN gem install fpm +# ---> Running in c2fc672985ea +# ERROR: Error installing fpm: +# ruby-xz requires Ruby version >= 1.9.3. +# So +# https://gist.github.com/slouma2000/8619039 +# and +# http://stackoverflow.com/questions/28129438/docker-rvm-command-not-found +RUN gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN curl -L get.rvm.io | bash -s stable +RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm install 1.9.3" +RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm use 1.9.3 --default" + +# Update rubygems +RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && gem update --system" +RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && gem install bundler" + +# install fpm +RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && gem install fpm" + +# install pip + dependencies +RUN yum -y install python-pip python-argparse +RUN pip install --upgrade pip + +# COPY script +# https://docs.docker.com/engine/reference/builder/#copy +COPY py2rpm.py $working_dir + +# ENTRYPOINT to generate RPMs +# https://docs.docker.com/engine/reference/builder/#entrypoint +# Invoke like: +# (lcars)# docker run 3e4251a30f93 m3u8 +# or +# docker run -v /Users/jehammel/foo/:/opt/out 3e4251a30f93 m3u8 -o /opt/out +ENTRYPOINT ["python", "py2rpm.py"]
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pypedream/__init__.py Mon Feb 06 03:40:08 2017 +0000 @@ -0,0 +1,2 @@ +# +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pypedream/eval.py Mon Feb 06 03:40:08 2017 +0000 @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +evaluate a bash script or command +""" + +# imports +import argparse +import os +import subprocess +import sys +import tempfile + + +def parse_env(output): + """parse environment content a la `env` output""" + + output = output.strip() + lines = output.splitlines() + return dict([line.split('=', 1) for line in lines]) + + +def source(src, update=True): + """ + eval a file and return the environment dictionary + """ + command = 'set -e && source "{src}" && env'.format(src=src) + output = subprocess.check_output(command, shell=True) + env = parse_env(output) + if update: + os.environ.update(env) + return env + + +def eval_command(command, update=True): + """ + evaluate a command + + command -- command to evaluate + update -- whether to update os.environ with result + """ + + output = subprocess.check_output(command, shell=True) + fd, name = tempfile.mkstemp() + try: + os.write(fd, output) + os.close(fd) + env = source(name, update=update) + finally: + try: + os.remove(name) + except: + pass + return env + + +def main(args=sys.argv[1:]): + + # parse command line + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', help="command to run") + options = parser.parse_args(args) + + # keep copy of the old env + old_env = os.environ.copy() + + # calcaulate the new environment + new_env = eval_command(options.command) + + # compute difference + updated = [] + for key, value in new_env.items(): + if value != old_env.get(key): + updated.append(key) + updated.sort() + + # output + for key in updated: + print ('{key}={value}'.format(key=key, value=new_env[key])) + + +if __name__ == '__main__': + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pypedream/py2centos.py Mon Feb 06 03:40:08 2017 +0000 @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +convert python packages to Centos(6) RPMs using Docker +""" + +# imports +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from eval import eval_command +from jinja2 import Template + +# module globals +here = os.path.dirname(os.path.realpath(__file__)) + + +def ensure_dir(directory): + """ensure a directory exists""" + + if os.path.exists(directory): + if not os.path.isdir(directory): + raise OSError("Not a directory: '{}'".format(directory)) + return directory + os.makedirs(directory) + return directory + + +def main(args=sys.argv[1:]): + """CLI""" + + # parse command line + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('package') + parser.add_argument('-o', '--output-dir', dest='output_dir', + type=ensure_dir, default='.', + help="directory for output [DEFAULT: %(default)s]") + parser.add_argument('--fpm', '--use-fpm', dest='use_fpm', + action='store_true', default=False, + help="use fpm for package conversion") + parser.add_argument('--version', '--centos-version', dest='version', + default='6', + help="Centos version to build under [DEFAULT: %(default)s]") + parser.add_argument('-v', '--verbose', dest='verbose', + action='store_true', default=False, + help="enable verbose output") + options = parser.parse_args(args) + + # load the Dockerfile template contents + dockerfile = os.path.join(here, 'Dockerfile') + with open(dockerfile, 'r') as f: + contents = f.read() + template = Template(contents) + + tempdir = tempfile.mkdtemp() + try: + # output the rendered template + output = os.path.join(tempdir, 'Dockerfile') + data = {'version': options.version} + with open(output, 'w') as f: + f.write(template.render(**data)) + + # copy the script in so that it may be picked up by `docker build` + # see https://docs.python.org/2/library/shutil.html + script = 'py2rpm.py' + src = os.path.join(here, script) + shutil.copy2(src, tempdir) + + # build the machine + # see also http://blog.endpoint.com/2015/01/getting-realtime-output-using-python.html + command = ['docker', 'build', '.'] + output = subprocess.check_output(command, cwd=tempdir) + last_line = output.strip().splitlines()[-1].strip() + print (last_line) + if not last_line.startswith('Success'): + raise AssertionError(last_line) + container = last_line.split()[-1] + if options.verbose: + print ("Container: {container}".format(container=container)) + except subprocess.CalledProcessError as e: + print ("Directory: {tempdir}".format(tempdir=tempdir)) + print ("{command} exited with {returncode}".format(command=e.cmd, + returncode=e.returncode)) + print (e.output) + import pdb; pdb.set_trace() + finally: + shutil.rmtree(tempdir) + + # run the container + mount the docker volume (and such) + # https://docs.docker.com/engine/userguide/containers/dockervolumes/#mount-a-host-directory-as-a-data-volume + # (lcars)# docker run -v /Users/jehammel/foo/:/opt/out 3e4251a30f93 m3u8 -o /opt/out + output_dir = os.path.abspath(options.output_dir) + command = ['docker', + 'run', + '-v', + '{0}:/opt/out'.format(output_dir), + container, + '-o', + '/opt/out', + options.package] + if options.use_fpm: + command.append('--use-fpm') + if options.verbose: + print ("Running:\n{command}".format(command=subprocess.list2cmdline(command))) + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + import pdb; pdb.set_trace() + print (subprocess.list2cmdline(command)) + raise + + +if __name__ == '__main__': + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pypedream/py2rpm.py Mon Feb 06 03:40:08 2017 +0000 @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +""" +convert a python package to RPM +""" + +# Links: +# - https://pypi.python.org/pypi/py2pack : generates spec files! However, requires pypi as a source :( +# - http://wheel.readthedocs.org/en/latest/ : untried! Not sure how it solves the problem +# - https://docs.python.org/2.0/dist/creating-rpms.html : Not very helpful! +# - https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.pip.html : Probably the way to go! +# * see also https://salt.readthedocs.org/en/v0.17.1/ref/states/all/salt.states.pip_state.html +# - https://github.com/jordansissel/fpm : Does all the things! However, it will use host not guest operating system :( and does not operate on files :( +# - http://blog.ziade.org/2011/03/25/bdist_rpm-is-dead-long-life-to-py2rpm/ : doesn't actually find any files! +# - https://github.com/fedora-python/pyp2rpm : Doesn't seem to work! + +# imports +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def ensure_dir(directory): + """ensure a directory exists""" + + if os.path.exists(directory): + if not os.path.isdir(directory): + raise OSError("Not a directory: '{}'".format(directory)) + + return directory + + os.makedirs(directory) + return directory + + +def filename2data(filename): + ext = '.tar.gz' + if not filename.endswith(ext): + raise NotImplementedError("""Oops! I didn't think you would see this! +I didn't have time to figure out how python packaging actually resolves + names of python packages, and for that, I apologize. So I only support +one format: .tar.gz""") + return filename.rsplit(ext)[0].split('-', 1) + + +def directory2packagedata(directory): + """expects directory to ONLY have packages in it""" + + retval = [] + for package in os.listdir(directory): + name, version = filename2data(package) + retval.append(dict(name=name, + version=version, + basename=package, + path=os.path.join(directory, package))) + return dict([(item['name'], item) + for item in retval]) + + +def untar(tarball, destination): + """untar a `tarball` to `destination`""" + # TODO: this can be done without shelling out + + command = ['tar', 'xzf', tarball, '-C', destination] + subprocess.check_call(command) + + +def bdist_rpm(directory): + """build a directory with a setup.py and return the RPM""" + + command = [sys.executable, 'setup.py', 'bdist_rpm'] + subprocess.check_call(command, cwd=directory) + dist = os.path.join(directory, 'dist') + contents = os.listdir(dist) + ext = '.noarch.rpm' + rpms = [f for f in contents + if f.endswith(ext)] + if len(rpms) != 1: + raise AssertionError("Expecting one RPM file, found {0}".format(len(rpms))) + return os.path.abspath(os.path.join(dist, rpms.pop())) + + +def fpm_converter(directory): + """ + use `fpm` within a directory and return the RPM + See: + https://github.com/jordansissel/fpm/wiki/ConvertingPython + """ + ext = '.rpm' + contents = os.listdir(directory) + assert 'setup.py' in contents + assert not bool([i for i in contents if i.endswith(ext)]) + command = ['fpm', '--rpm-os', 'linux', '-s', 'python', '-t', 'rpm', 'setup.py'] + subprocess.check_call(command, cwd=directory) + contents = os.listdir(directory) + rpms = [i for i in contents if i.endswith(ext)] + if len(rpms) != 1: + raise AssertionError("Expecting one RPM file, found {0}".format(len(rpms))) + return os.path.abspath(os.path.join(directory, rpms.pop())) + + +def main(args=sys.argv[1:]): + """CLI""" + + # parse command line + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('package') + parser.add_argument('-o', '--output-dir', dest='output_dir', + type=ensure_dir, default='.', + help="directory for output [DEFAULT: %(default)s]") + parser.add_argument('--fpm', '--use-fpm', dest='use_fpm', + action='store_true', default=False, + help="use fpm for package conversion") + parser.add_argument('-v', '--verbose', dest='verbose', + action='store_true', default=False, + help="be verbose") + options = parser.parse_args(args) + + # make a temporary directory + tmpdir = tempfile.mkdtemp() + try: + + # `pip` command to download the package: + # http://stackoverflow.com/questions/7300321/how-to-use-pythons-pip-to-download-and-keep-the-zipped-files-for-a-package + command = ['pip', 'install', + '--download', tmpdir, + options.package, + '--no-use-wheel'] + try: + subprocess.check_call(command) + except Exception as e: + print subprocess.list2cmdline(command) + raise + + # get the package data + packagedata = directory2packagedata(tmpdir) + if options.verbose: + print (json.dumps(packagedata, sort_keys=True, indent=2)) + + # ensure we have the one package we specified; + # the rest are dependencies + if options.package not in packagedata: + raise AssertionError("Package '{}' not in packagedata: {}".format(options.package, json.dumps(packagedata))) + + # build each package + for package, metadata in packagedata.items(): + path = metadata['path'] + print (package) + print (path) + + # make a `build` directory + builddir = tempfile.mkdtemp() + try: + # inflate the tarball + untar(path, builddir) + contents = os.listdir(builddir) + print (",".join(contents)) + if len(contents) != 1: + raise AssertionError("Ooops!") + + # find the package directory + subdir = os.path.join(builddir, contents.pop()) + if not os.path.isdir(subdir): + raise AssertionError("Ooops!") + contents = os.listdir(subdir) + print (",".join(contents)) + if 'setup.py' not in contents: + raise AssertionError("Cannot find setup.py") + + # build the rpm + if options.use_fpm: + rpm = fpm_converter(subdir) + else: + rpm = bdist_rpm(subdir) + + # move it to output directory + shutil.move(rpm, options.output_dir) + + finally: + # cleanup + if True: + shutil.rmtree(builddir, ignore_errors=True) + else: + print ("build directory: {}".format(builddir)) + + finally: + # cleanup + if True: + shutil.rmtree(tmpdir, ignore_errors=True) + else: + # print tmpdir contents + print ("{} :".format(tmpdir)) + print ('\n'.join(sorted(os.listdir(tmpdir)))) + +if __name__ == '__main__': + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.py Mon Feb 06 03:40:08 2017 +0000 @@ -0,0 +1,45 @@ +""" +setup packaging script for pypedream +""" + +import os + +version = "0.3" +dependencies = ['PyYAML', + 'Jinja2'] + +# allow use of setuptools/distribute or distutils +kw = {} +try: + from setuptools import setup + kw['entry_points'] = """ + [console_scripts] + py2rpm = pypedream.py2rpm:main + py2centos = pypedream.py2centos:main +""" + kw['install_requires'] = dependencies +except ImportError: + from distutils.core import setup + kw['requires'] = dependencies + +try: + here = os.path.dirname(os.path.abspath(__file__)) + description = file(os.path.join(here, 'README.txt')).read() +except IOError: + description = '' + + +setup(name='pypedream', + version=version, + description="python packaging work", + long_description=description, + classifiers=[], # Get strings from http://www.python.org/pypi?%3Aaction=list_classifiers + author='Jeff Hammel', + author_email='k0scist@gmail.com', + url='', + license='', + packages=['pypedream'], + include_package_data=True, + zip_safe=False, + **kw + )