Mercurial > hg > PaInt
annotate paint/package.py @ 29:59524b6d25c0
implementing making pypi directory strcuture from a package
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Fri, 30 Mar 2012 11:41:04 -0700 |
| parents | c44fb4b6918b |
| children | fe5f282dca9b |
| rev | line source |
|---|---|
| 4 | 1 """ |
| 2 package model for python PAckage INTrospection | |
| 3 """ | |
| 4 | |
| 7 | 5 import os |
| 28 | 6 import pip |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
7 import pypi |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
8 import shutil |
| 19 | 9 import subprocess |
| 10 import sys | |
| 7 | 11 import tarfile |
| 5 | 12 import tempfile |
| 7 | 13 import urllib2 |
|
21
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
14 import urlparse |
| 7 | 15 import utils |
| 5 | 16 |
| 19 | 17 try: |
| 18 from subprocess import check_call as call | |
| 19 except ImportError: | |
| 20 from subporcess import call | |
| 21 | |
| 10 | 22 __all__ = ['Package'] |
| 23 | |
| 4 | 24 class Package(object): |
| 25 | |
| 26 def __init__(self, src): | |
| 27 self.src = src | |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
28 |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
29 # ephemeral data |
| 5 | 30 self._tmppath = None |
| 28 | 31 self._egg_info_path = None |
| 5 | 32 |
| 33 def path(self): | |
| 34 """filesystem path to package""" | |
| 7 | 35 |
| 36 # return cached copy if it exists | |
| 37 if self._tmppath: | |
| 38 return self._tmppath | |
| 39 | |
| 40 # fetch from the web if a URL | |
| 41 tmpfile = None | |
| 42 src = self.src | |
| 5 | 43 if utils.isURL(self.src): |
| 8 | 44 tmpfile = src = self.fetch() |
| 7 | 45 |
| 46 # unpack if an archive | |
| 28 | 47 if self._is_archive(src): |
| 8 | 48 try: |
| 49 self.unpack(src) | |
| 50 finally: | |
| 51 if tmpfile: | |
| 52 os.remove(tmpfile) | |
| 7 | 53 return self._tmppath |
| 54 | |
| 5 | 55 return self.src |
| 56 | |
| 57 def fetch(self): | |
| 8 | 58 """fetch from remote source to a temporary file""" |
|
15
8c8b7482772f
fix a few failures but still failing
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
59 resource = urllib2.urlopen(self.src) |
| 11 | 60 fd, filename = tempfile.mkstemp() |
| 7 | 61 os.write(fd, resource.read()) |
| 62 os.close(fd) | |
| 63 return filename | |
| 64 | |
| 65 def unpack(self, archive): | |
| 66 """unpack the archive to a temporary destination""" | |
| 67 # TODO: should handle zipfile additionally at least | |
| 68 # Ideally, this would be pluggable, etc | |
| 8 | 69 assert tarfile.is_tarfile(archive), "%s is not an archive" % self.src |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
70 tf = tarfile.TarFile.open(archive) |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
71 self._tmppath = tempfile.mkdtemp() |
| 19 | 72 members = tf.getmembers() |
| 73 | |
| 74 # cut off the top level directory | |
| 75 assert not [i for i in members if not os.path.sep in i.name] | |
| 76 tld = set() | |
| 77 for member in members: | |
| 78 directory, member.name = member.name.split(os.path.sep, 1) | |
| 79 tld.add(directory) | |
| 80 assert len(tld) == 1 | |
| 81 | |
| 82 # extract | |
| 83 for member in members: | |
| 84 tf.extract(member, path=self._tmppath) | |
| 85 tf.close() | |
| 7 | 86 |
| 28 | 87 def _is_archive(self, path): |
| 7 | 88 """returns if the filesystem path is an archive""" |
| 89 # TODO: should handle zipfile additionally at least | |
| 90 # Ideally, this would be pluggable, etc | |
| 91 return tarfile.is_tarfile(path) | |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
92 |
| 28 | 93 def _cleanup(self): |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
94 if self._tmppath: |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
95 shutil.rmtree(self._tmppath) |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
96 self._tmppath = None |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
97 |
|
18
4c4d1e194bf7
leave in debugging code for now
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
98 # __del__ = cleanup |
| 11 | 99 |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
100 ### python-package-specific functionality |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
101 |
| 28 | 102 def _egg_info(self): |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
103 """build the egg_info directory""" |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
104 |
| 28 | 105 if self._egg_info_path: |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
106 # return cached copy |
| 28 | 107 return self._egg_info_path |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
108 |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
109 directory = self.path() |
| 16 | 110 setup_py = os.path.join(directory, 'setup.py') |
| 111 if not os.path.exists(setup_py): | |
| 17 | 112 raise AssertionError("%s does not exist" % setup_py) |
| 11 | 113 |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
114 # setup the egg info |
| 19 | 115 call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=subprocess.PIPE) |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
116 |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
117 # get the .egg-info directory |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
118 egg_info = [i for i in os.listdir(directory) |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
119 if i.endswith('.egg-info')] |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
120 assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info) |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
121 egg_info = os.path.join(directory, egg_info[0]) |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
122 assert os.path.isdir(egg_info), "%s is not a directory" % egg_info |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
123 |
|
14
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
124 # cache it |
| 28 | 125 self._egg_info_path = egg_info |
| 126 return self._egg_info_path | |
|
14
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
127 |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
128 def info(self): |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
129 """return info dictionary for package""" |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
130 # could use pkginfo |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
131 |
| 28 | 132 egg_info = self._egg_info() |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
133 |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
134 # read the package information |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
135 pkg_info = os.path.join(egg_info, 'PKG-INFO') |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
136 info_dict = {} |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
137 for line in file(pkg_info).readlines(): |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
138 if not line or line[0].isspace(): |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
139 continue # XXX neglects description |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
140 assert ':' in line |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
141 key, value = [i.strip() for i in line.split(':', 1)] |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
142 info_dict[key] = value |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
143 |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
144 # return the information |
|
20
11bc01a089e7
return only the dictionary
Jeff Hammel <jhammel@mozilla.com>
parents:
19
diff
changeset
|
145 return info_dict |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
146 |
| 11 | 147 def dependencies(self): |
|
14
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
148 """return the dependencies""" |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
149 |
|
21
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
150 # get the egg_info directory |
| 28 | 151 egg_info = self._egg_info() |
|
14
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
152 |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
153 # read the dependencies |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
154 requires = os.path.join(egg_info, 'requires.txt') |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
155 if os.path.exists(requires): |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
156 dependencies = [i.strip() for i in file(requires).readlines() if i.strip()] |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
157 else: |
|
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
158 dependencies = [] |
|
21
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
159 dependencies = dict([(i, None) for i in dependencies]) |
|
14
6d27c2136163
a decent structure; need to reuse more
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
160 |
|
21
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
161 # read the dependency links |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
162 dependency_links = os.path.join(egg_info, 'dependency_links.txt') |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
163 if os.path.exists(dependency_links): |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
164 links = [i.strip() for i in file(dependency_links).readlines() if i.strip()] |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
165 for link in links: |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
166 # XXX pretty ghetto |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
167 assert '#egg=' in link |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
168 url, dep = link.split('#egg=', 1) |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
169 if dep in dependencies: |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
170 dependencies[dep] = link |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
171 |
|
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
172 return dependencies |
| 24 | 173 |
| 174 def download(self, directory): | |
| 175 """download a package and all its dependencies using pip""" | |
| 28 | 176 if not os.path.exists(directory): |
| 177 os.makedirs(directory) | |
| 178 assert os.path.isdir(directory) | |
| 179 pip.main(['install', '--download', directory, self.src]) | |
| 180 | |
| 181 def pypi(self, directory): | |
| 182 """ | |
| 183 download packages for a pypi directory structure | |
| 184 http://k0s.org/portfolio/pypi.html | |
| 185 """ | |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
186 if not os.path.exists(directory): |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
187 os.makedirs(directory) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
188 assert os.path.isdir(directory) |
| 28 | 189 tempdir = tempfile.mkdtemp() |
| 190 try: | |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
191 self.download(tempdir) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
192 for package in os.listdir(tempdir): |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
193 |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
194 # full path |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
195 src = os.path.join(tempdir, package) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
196 |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
197 # get destination dirname, filename |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
198 dirname, filename = pypi.pypi_path(src) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
199 |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
200 # make the directory if it doesn't exist |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
201 subdir = os.path.join(directory, dirname) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
202 if not os.path.exists(subdir): |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
203 os.path.makedirs(subdir) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
204 assert os.path.isdir(subdir) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
205 |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
206 # move the file |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
207 shutil.move(src, os.path.join(subdir, filename)) |
| 28 | 208 finally: |
| 209 shutil.rmtree(tempdir) |