Mercurial > hg > PaInt
annotate paint/package.py @ 69:fea269259222
move comment to more appropriate place
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Sat, 26 Jan 2013 00:00:56 -0800 |
| parents | d001017d5870 |
| children | da69d58f960d |
| rev | line source |
|---|---|
| 4 | 1 """ |
| 2 package model for python PAckage INTrospection | |
| 3 """ | |
| 4 | |
| 36 | 5 # TODO: use pkginfo.sdist more |
| 6 | |
| 57 | 7 import info |
| 7 | 8 import os |
| 28 | 9 import pip |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
10 import shutil |
| 19 | 11 import subprocess |
| 12 import sys | |
| 7 | 13 import tarfile |
| 5 | 14 import tempfile |
| 7 | 15 import urllib2 |
|
21
4df3e715817d
now have passing tests again
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
16 import urlparse |
| 7 | 17 import utils |
| 5 | 18 |
|
62
726008cff4e3
remove code supporting old version
Jeff Hammel <jhammel@mozilla.com>
parents:
58
diff
changeset
|
19 from subprocess import check_call as call |
| 19 | 20 |
| 10 | 21 __all__ = ['Package'] |
| 22 | |
| 4 | 23 class Package(object): |
| 33 | 24 """ |
| 25 class for python package introspection. | |
| 26 constructor takes the package 'src' | |
| 27 """ | |
| 4 | 28 |
| 50 | 29 def __init__(self, src, verbose=True): |
| 63 | 30 """ |
| 31 - src : URL or filesystem path to the package | |
| 32 """ | |
| 4 | 33 self.src = src |
| 50 | 34 self.verbose = verbose |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
35 |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
36 # ephemeral data |
| 5 | 37 self._tmppath = None |
| 28 | 38 self._egg_info_path = None |
| 33 | 39 self._build_path = None |
|
40
06f21791eba1
add yet another indirection layer
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
40 self._pkg_info_path = None |
| 5 | 41 |
|
64
1a279bac0afa
start moving this monster to a package information provider
Jeff Hammel <jhammel@mozilla.com>
parents:
63
diff
changeset
|
42 # package metadata provider |
|
1a279bac0afa
start moving this monster to a package information provider
Jeff Hammel <jhammel@mozilla.com>
parents:
63
diff
changeset
|
43 # TODO: this should be chooseable but currently the keys of the |
|
1a279bac0afa
start moving this monster to a package information provider
Jeff Hammel <jhammel@mozilla.com>
parents:
63
diff
changeset
|
44 # interfaces are not the same |
|
1a279bac0afa
start moving this monster to a package information provider
Jeff Hammel <jhammel@mozilla.com>
parents:
63
diff
changeset
|
45 self.package_info = info.EggInfo |
|
1a279bac0afa
start moving this monster to a package information provider
Jeff Hammel <jhammel@mozilla.com>
parents:
63
diff
changeset
|
46 |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
47 # TODO: list of temporary files/directories to be deleted |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
48 |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
49 def _log(self, message): |
|
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
50 if self.verbose: |
|
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
51 print '>>> %s' % message |
|
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
52 |
| 34 | 53 def _path(self): |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
54 """filesystem path to package directory""" |
| 7 | 55 |
| 51 | 56 self._log(">>> _path:Getting path to package") |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
57 |
| 7 | 58 # return cached copy if it exists |
| 59 if self._tmppath: | |
| 60 return self._tmppath | |
| 61 | |
| 62 # fetch from the web if a URL | |
| 63 tmpfile = None | |
| 64 src = self.src | |
| 5 | 65 if utils.isURL(self.src): |
| 8 | 66 tmpfile = src = self.fetch() |
| 7 | 67 |
| 68 # unpack if an archive | |
| 28 | 69 if self._is_archive(src): |
| 8 | 70 try: |
| 37 | 71 self._unpack(src) |
| 8 | 72 finally: |
| 73 if tmpfile: | |
| 74 os.remove(tmpfile) | |
| 7 | 75 return self._tmppath |
| 76 | |
| 5 | 77 return self.src |
| 78 | |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
79 def fetch(self, filename=None): |
| 8 | 80 """fetch from remote source to a temporary file""" |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
81 if filename is None: |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
82 fd, filename = tempfile.mkstemp() |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
83 os.close(fd) |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
84 fp = file(filename, 'w') |
|
15
8c8b7482772f
fix a few failures but still failing
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
85 resource = urllib2.urlopen(self.src) |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
86 fp.write(resource.read()) |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
87 fp.close() |
| 7 | 88 return filename |
| 89 | |
| 37 | 90 def _unpack(self, archive): |
| 7 | 91 """unpack the archive to a temporary destination""" |
| 92 # TODO: should handle zipfile additionally at least | |
| 93 # Ideally, this would be pluggable, etc | |
| 8 | 94 assert tarfile.is_tarfile(archive), "%s is not an archive" % self.src |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
95 tf = tarfile.TarFile.open(archive) |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
96 self._tmppath = tempfile.mkdtemp() |
| 19 | 97 members = tf.getmembers() |
| 98 | |
| 99 # cut off the top level directory | |
| 37 | 100 members = [i for i in members if os.path.sep in i.name] |
| 19 | 101 tld = set() |
| 102 for member in members: | |
| 103 directory, member.name = member.name.split(os.path.sep, 1) | |
| 104 tld.add(directory) | |
| 105 assert len(tld) == 1 | |
| 106 | |
| 107 # extract | |
| 108 for member in members: | |
| 109 tf.extract(member, path=self._tmppath) | |
| 110 tf.close() | |
| 7 | 111 |
| 28 | 112 def _is_archive(self, path): |
| 7 | 113 """returns if the filesystem path is an archive""" |
| 114 # TODO: should handle zipfile additionally at least | |
| 115 # Ideally, this would be pluggable, etc | |
| 116 return tarfile.is_tarfile(path) | |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
117 |
| 28 | 118 def _cleanup(self): |
|
9
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
119 if self._tmppath: |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
120 shutil.rmtree(self._tmppath) |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
121 self._tmppath = None |
|
562230cc2e86
now the package class should at least extracts things. it doesnt actually do anything useful
Jeff Hammel <jhammel@mozilla.com>
parents:
8
diff
changeset
|
122 |
| 65 | 123 __del__ = _cleanup |
| 11 | 124 |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
125 ### python-package-specific functionality |
|
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
126 |
|
40
06f21791eba1
add yet another indirection layer
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
127 def info(self): |
|
06f21791eba1
add yet another indirection layer
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
128 """return info dictionary for package""" |
|
53
c588375a7ce4
starting a new strawman to work out kinks
Jeff Hammel <jhammel@mozilla.com>
parents:
52
diff
changeset
|
129 self._log(">>> Getting the info") |
|
66
af7609457dc6
introduce a failing test; yay!
Jeff Hammel <jhammel@mozilla.com>
parents:
65
diff
changeset
|
130 return self.package_info(self._path())() |
|
13
0dd1f8f83be2
moving towards some sort of organization
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
131 |
| 11 | 132 def dependencies(self): |
|
68
d001017d5870
migrate this code to the right place
Jeff Hammel <jhammel@mozilla.com>
parents:
66
diff
changeset
|
133 return self.package_info(self._path()).dependencies() |
| 24 | 134 |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
135 def extension(self): |
| 35 | 136 """filename extension of the package""" |
| 137 | |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
138 print ">>> extension:Getting package" |
| 35 | 139 package = self.package() |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
140 print ">>> extension:package=%s" % package |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
141 |
| 35 | 142 # determine the extension (XXX hacky) |
| 143 extensions = ('.tar.gz', '.zip', '.tar.bz2') | |
| 144 for ext in extensions: | |
| 36 | 145 if package.endswith(ext): |
| 35 | 146 return ext |
| 147 | |
| 148 raise Exception("Extension %s not found: %s" % (extensions, package)) | |
| 149 | |
| 150 def package(self, destination=None): | |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
151 """ |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
152 repackage the package to ensure its actually in the right form |
| 34 | 153 and return the path to the destination |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
154 - destination: if given, path to put the build in |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
155 """ |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
156 |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
157 self._log("package: Getting package directory, destination=%s" % repr(destination)) |
|
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
158 |
| 33 | 159 if self._build_path: |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
160 self._log("package: build_path already set: %s" % self._build_path) |
| 34 | 161 if destination: |
| 162 shutil.copy(self._build_path, destination) | |
| 163 return os.path.abspath(destination) | |
| 164 | |
| 165 # return cached copy | |
| 33 | 166 return self._build_path |
| 167 | |
| 34 | 168 path = self._path() |
| 169 dist = os.path.join(path, 'dist') | |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
170 self._log("package: dist directory: %s; (path=%s)" % (dist, path)) |
| 34 | 171 if os.path.exists(dist): |
| 172 shutil.rmtree(dist) | |
| 173 | |
|
49
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
174 command = [sys.executable, 'setup.py', 'sdist'] |
|
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
175 self._log("package: running: %s" % ' '.join(command)) |
|
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
176 call(command, cwd=path) |
|
85374a69cf89
call in foreground to avoid hang :(
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
177 self._log("package: done running setup.py dist") |
| 34 | 178 |
| 179 assert os.path.exists(dist) | |
| 180 contents = os.listdir(dist) | |
| 181 assert len(contents) == 1 | |
| 182 | |
| 183 self._build_path = os.path.join(dist, contents[0]) | |
| 184 | |
| 185 # destination | |
| 186 # use an evil recursive trick | |
| 187 if destination: | |
| 36 | 188 return self.package(destination=destination) |
| 34 | 189 |
| 190 return self._build_path | |
| 191 | |
| 24 | 192 def download(self, directory): |
| 193 """download a package and all its dependencies using pip""" | |
| 28 | 194 if not os.path.exists(directory): |
| 195 os.makedirs(directory) | |
| 196 assert os.path.isdir(directory) | |
| 197 pip.main(['install', '--download', directory, self.src]) | |
| 198 | |
| 199 def pypi(self, directory): | |
| 200 """ | |
| 201 download packages for a pypi directory structure | |
| 202 http://k0s.org/portfolio/pypi.html | |
| 203 """ | |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
204 if not os.path.exists(directory): |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
205 os.makedirs(directory) |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
206 assert os.path.isdir(directory) |
| 28 | 207 tempdir = tempfile.mkdtemp() |
| 208 try: | |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
209 self.download(tempdir) |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
210 files = os.listdir(tempdir) |
| 52 | 211 self._log(">>> Files: %s" % files) |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
212 for f in files: |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
213 |
|
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
214 # full path |
| 47 | 215 src = os.path.join(tempdir, f) |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
216 |
| 35 | 217 # make a package of the thing |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
218 print ">>> pypi:Packaging %s" % src |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
219 package = Package(src) |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
220 print ">>> pypi:DONE packaging %s" % src |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
221 |
| 36 | 222 # get destination dirname, filename |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
223 print ">>> pypi:Getting PyPI path" |
| 36 | 224 dirname, filename = package.pypi_path() |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
225 print ">>> pypi:DONE PyPI path: %s/%s" % (dirname, filename) |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
226 |
| 36 | 227 # make the directory if it doesn't exist |
| 228 subdir = os.path.join(directory, dirname) | |
| 229 if not os.path.exists(subdir): | |
| 230 os.makedirs(subdir) | |
| 231 assert os.path.isdir(subdir) | |
|
29
59524b6d25c0
implementing making pypi directory strcuture from a package
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
232 |
| 36 | 233 # move the file |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
234 print ">>> pypi:Moving to PyPI path %s/%s" % (subdir, filename) |
| 47 | 235 package.package(destination=os.path.join(subdir, filename)) |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
236 print ">>> Done with %s" % src |
| 28 | 237 finally: |
| 238 shutil.rmtree(tempdir) | |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
239 |
| 36 | 240 def pypi_path(self): |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
241 """ |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
242 returns subpath 2-tuple appropriate for pypi path structure: |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
243 http://k0s.org/portfolio/pypi.html |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
244 """ |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
245 print ">>> pypi_path:Getting info" |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
246 info = self.info() |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
247 print ">>> pypi_path:DONE getting info" |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
248 |
| 36 | 249 # determine the extension |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
250 print ">>> pypi_path:Getting extension" |
| 36 | 251 extension = self.extension() |
|
48
abfb267e8332
debugging info; lord i hate this
Jeff Hammel <jhammel@mozilla.com>
parents:
47
diff
changeset
|
252 print ">>> pypi_path:DONE Getting extension: %s" % extension |
|
31
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
253 |
|
5fb1844db0b2
back to the drawring broad
Jeff Hammel <jhammel@mozilla.com>
parents:
30
diff
changeset
|
254 # get the filename destination |
| 36 | 255 name = info['Name'] |
| 256 version = info['Version'] | |
| 257 filename = '%s-%s%s' % (name, version, extension) | |
| 258 return name, filename |