View Javadoc
1   package org.argeo.slc.repo.osgi;
2   
3   import java.io.ByteArrayInputStream;
4   import java.io.ByteArrayOutputStream;
5   import java.io.InputStream;
6   import java.util.ArrayList;
7   import java.util.HashMap;
8   import java.util.Iterator;
9   import java.util.List;
10  import java.util.Map;
11  import java.util.Set;
12  import java.util.TreeSet;
13  import java.util.jar.JarInputStream;
14  import java.util.zip.ZipEntry;
15  import java.util.zip.ZipInputStream;
16  import java.util.zip.ZipOutputStream;
17  
18  import javax.jcr.Node;
19  import javax.jcr.Property;
20  import javax.jcr.RepositoryException;
21  import javax.jcr.Session;
22  
23  import org.apache.commons.io.FilenameUtils;
24  import org.apache.commons.io.IOUtils;
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.argeo.jcr.JcrUtils;
28  import org.argeo.slc.CategorizedNameVersion;
29  import org.argeo.slc.DefaultNameVersion;
30  import org.argeo.slc.ModuleSet;
31  import org.argeo.slc.NameVersion;
32  import org.argeo.slc.SlcException;
33  import org.argeo.slc.build.Distribution;
34  import org.argeo.slc.build.License;
35  import org.argeo.slc.repo.OsgiFactory;
36  import org.argeo.slc.repo.RepoUtils;
37  import org.argeo.slc.repo.maven.ArtifactIdComparator;
38  import org.eclipse.aether.artifact.Artifact;
39  import org.eclipse.aether.artifact.DefaultArtifact;
40  import org.springframework.util.AntPathMatcher;
41  import org.springframework.util.PathMatcher;
42  
43  import aQute.bnd.osgi.Jar;
44  
45  /**
46   * Download a software distribution and generates the related OSGi bundles from
47   * the jars, or import them directly if they are already OSGi bundles and don't
48   * need further modification.
49   */
50  public class ArchiveWrapper implements Runnable, ModuleSet, Distribution {
51  	private final static Log log = LogFactory.getLog(ArchiveWrapper.class);
52  
53  	private OsgiFactory osgiFactory;
54  	private String version;
55  	private License license;
56  
57  	private String uri;
58  
59  	/** Jars to wrap as OSGi bundles */
60  	private Map<String, BndWrapper> wrappers = new HashMap<String, BndWrapper>();
61  
62  	private SourcesProvider sourcesProvider;
63  
64  	// pattern of OSGi bundles to import
65  	private PathMatcher pathMatcher = new AntPathMatcher();
66  	private Map<String, String> includes = new HashMap<String, String>();
67  	private List<String> excludes = new ArrayList<String>();
68  
69  	private Boolean mavenGroupIndexes = false;
70  
71  	public void init() {
72  		for (BndWrapper wrapper : wrappers.values()) {
73  			wrapper.setFactory(this);
74  			if (version != null && wrapper.getVersion() == null)
75  				wrapper.setVersion(version);
76  			if (license != null && wrapper.getLicense() == null)
77  				wrapper.setLicense(license);
78  		}
79  	}
80  
81  	public void destroy() {
82  
83  	}
84  
85  	public String getDistributionId() {
86  		return uri;
87  	}
88  
89  	public Iterator<? extends NameVersion> nameVersions() {
90  		if (wrappers.size() > 0)
91  			return wrappers.values().iterator();
92  		else
93  			return osgiNameVersions();
94  	}
95  
96  	@SuppressWarnings("resource")
97  	protected Iterator<? extends NameVersion> osgiNameVersions() {
98  		List<CategorizedNameVersion> nvs = new ArrayList<CategorizedNameVersion>();
99  
100 		Session distSession = null;
101 		ZipInputStream zin = null;
102 		try {
103 			distSession = osgiFactory.openDistSession();
104 
105 			Node distNode = osgiFactory.getDist(distSession, uri);
106 			zin = new ZipInputStream(
107 					distNode.getNode(Node.JCR_CONTENT).getProperty(Property.JCR_DATA).getBinary().getStream());
108 
109 			ZipEntry zentry = null;
110 			entries: while ((zentry = zin.getNextEntry()) != null) {
111 				String name = zentry.getName();
112 				if (log.isTraceEnabled())
113 					log.trace("Zip entry " + name);
114 				for (String exclude : excludes)
115 					if (pathMatcher.match(exclude, name))
116 						continue entries;
117 
118 				for (String include : includes.keySet()) {
119 					if (pathMatcher.match(include, name)) {
120 						String groupId = includes.get(include);
121 						JarInputStream jis = new JarInputStream(zin);
122 						if (jis.getManifest() == null) {
123 							log.warn("No MANIFEST in entry " + name + ", skipping...");
124 							continue entries;
125 						}
126 						NameVersion nv = RepoUtils.readNameVersion(jis.getManifest());
127 						if (nv != null) {
128 							if (nv.getName().endsWith(".source"))
129 								continue entries;
130 							CategorizedNameVersion cnv = new OsgiCategorizedNV(groupId, nv.getName(), nv.getVersion(),
131 									this);
132 							nvs.add(cnv);
133 							// no need to process further includes
134 							continue entries;
135 						}
136 					}
137 				}
138 			}
139 			return nvs.iterator();
140 		} catch (Exception e) {
141 			throw new SlcException("Cannot wrap distribution " + uri, e);
142 		} finally {
143 			IOUtils.closeQuietly(zin);
144 			JcrUtils.logoutQuietly(distSession);
145 		}
146 	}
147 
148 	public void run() {
149 		if (mavenGroupIndexes && (version == null))
150 			throw new SlcException("'mavenGroupIndexes' requires 'version' to be set");
151 
152 		Map<String, Set<Artifact>> binaries = new HashMap<String, Set<Artifact>>();
153 		Map<String, Set<Artifact>> sources = new HashMap<String, Set<Artifact>>();
154 
155 		Session distSession = null;
156 		Session javaSession = null;
157 		ZipInputStream zin = null;
158 		try {
159 			javaSession = osgiFactory.openJavaSession();
160 			distSession = osgiFactory.openDistSession();
161 
162 			if (log.isDebugEnabled())
163 				log.debug("Wrapping " + uri);
164 			boolean nothingWasDone = true;
165 
166 			Node distNode = osgiFactory.getDist(distSession, uri);
167 			zin = new ZipInputStream(
168 					distNode.getNode(Node.JCR_CONTENT).getProperty(Property.JCR_DATA).getBinary().getStream());
169 
170 			ZipEntry zentry = null;
171 			entries: while ((zentry = zin.getNextEntry()) != null) {
172 				String name = zentry.getName();
173 
174 				// sources autodetect
175 				String baseName = FilenameUtils.getBaseName(name);
176 				if (baseName.endsWith("-sources")) {
177 					String bundle = baseName.substring(0, baseName.length() - "-sources".length());
178 					// log.debug(name + "," + baseName + ", " + bundle);
179 					String bundlePath = FilenameUtils.getPath(name) + bundle + ".jar";
180 					if (wrappers.containsKey(bundlePath)) {
181 						BndWrapper wrapper = wrappers.get(bundlePath);
182 						NameVersion bundleNv = new DefaultNameVersion(wrapper.getName(), wrapper.getVersion());
183 						byte[] pdeSource = RepoUtils.packageAsPdeSource(zin, bundleNv);
184 						Artifact sourcesArtifact = new DefaultArtifact(wrapper.getCategory(),
185 								wrapper.getName() + ".source", "jar", wrapper.getVersion());
186 						Node pdeSourceNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), sourcesArtifact,
187 								pdeSource);
188 						osgiFactory.indexNode(pdeSourceNode);
189 						pdeSourceNode.getSession().save();
190 						if (log.isDebugEnabled())
191 							log.debug("Added sources " + sourcesArtifact + " for bundle " + wrapper.getArtifact()
192 									+ "from " + name + " in binary archive.");
193 					}
194 
195 				}
196 				// else if (baseName.endsWith(".source")) {
197 				// }
198 
199 				// binaries
200 				if (wrappers.containsKey(name)) {
201 					BndWrapper wrapper = (BndWrapper) wrappers.get(name);
202 					// we must copy since the stream is closed by BND
203 					byte[] origJarBytes = IOUtils.toByteArray(zin);
204 					Artifact artifact = wrapZipEntry(javaSession, zentry, origJarBytes, wrapper);
205 					nothingWasDone = false;
206 					addArtifactToIndex(binaries, wrapper.getGroupId(), artifact);
207 				} else {
208 					for (String wrapperKey : wrappers.keySet())
209 						if (pathMatcher.match(wrapperKey, name)) {
210 							// first matched is taken
211 							BndWrapper wrapper = (BndWrapper) wrappers.get(wrapperKey);
212 							// we must copy since the stream is closed by BND
213 							byte[] origJarBytes = IOUtils.toByteArray(zin);
214 							Artifact artifact = wrapZipEntry(javaSession, zentry, origJarBytes, wrapper);
215 							nothingWasDone = false;
216 							addArtifactToIndex(binaries, wrapper.getGroupId(), artifact);
217 							continue entries;
218 						} else {
219 							if (log.isTraceEnabled())
220 								log.trace(name + " not matched by " + wrapperKey);
221 						}
222 
223 					for (String exclude : excludes)
224 						if (pathMatcher.match(exclude, name))
225 							continue entries;
226 
227 					for (String include : includes.keySet()) {
228 						if (pathMatcher.match(include, name)) {
229 							String groupId = includes.get(include);
230 							byte[] origJarBytes = IOUtils.toByteArray(zin);
231 							Artifact artifact = importZipEntry(javaSession, zentry, origJarBytes, groupId);
232 							if (artifact == null) {
233 								log.warn("Skipped non identified " + zentry);
234 								continue entries;
235 							}
236 							nothingWasDone = false;
237 							if (artifact.getArtifactId().endsWith(".source"))
238 								addArtifactToIndex(sources, groupId, artifact);
239 							else
240 								addArtifactToIndex(binaries, groupId, artifact);
241 							// no need to process this entry further
242 							continue entries;
243 						}
244 					}
245 				}
246 			}
247 
248 			// indexes
249 			if (mavenGroupIndexes && version != null) {
250 				for (String groupId : binaries.keySet()) {
251 					RepoUtils.writeGroupIndexes(javaSession, "/", groupId, version, binaries.get(groupId),
252 							sources.containsKey(groupId) ? sources.get(groupId) : null);
253 				}
254 			}
255 
256 			if (nothingWasDone) {
257 				log.error("Nothing was done when wrapping " + uri + ". THE DISTRIBUTION IS INCONSISTENT.");
258 				// throw new SlcException("Nothing was done");
259 				// TODO Fail if not all wrappers matched
260 			}
261 
262 		} catch (Exception e) {
263 			throw new SlcException("Cannot wrap distribution " + uri, e);
264 		} finally {
265 			IOUtils.closeQuietly(zin);
266 			JcrUtils.logoutQuietly(distSession);
267 			JcrUtils.logoutQuietly(javaSession);
268 		}
269 	}
270 
271 	protected Artifact wrapZipEntry(Session javaSession, ZipEntry zentry, byte[] origJarBytes, BndWrapper wrapper)
272 			throws RepositoryException {
273 		ByteArrayOutputStream out = null;
274 		ByteArrayInputStream in = null;
275 		Node newJarNode;
276 		Jar jar = null;
277 		try {
278 			out = new ByteArrayOutputStream((int) zentry.getSize());
279 			in = new ByteArrayInputStream(origJarBytes);
280 			wrapper.wrapJar(in, out);
281 
282 			Artifact artifact = wrapper.getArtifact();
283 			newJarNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), artifact, out.toByteArray());
284 			osgiFactory.indexNode(newJarNode);
285 			newJarNode.getSession().save();
286 			if (log.isDebugEnabled())
287 				log.debug("Wrapped jar " + zentry.getName() + " to " + newJarNode.getPath());
288 
289 			if (sourcesProvider != null)
290 				addSource(javaSession, artifact, out.toByteArray());
291 
292 			return artifact;
293 		} finally {
294 			IOUtils.closeQuietly(in);
295 			IOUtils.closeQuietly(out);
296 			if (jar != null)
297 				jar.close();
298 		}
299 	}
300 
301 	protected void addSource(Session javaSession, Artifact artifact, byte[] binaryJarBytes) {
302 		InputStream in = null;
303 		ByteArrayOutputStream out = null;
304 		Jar jar = null;
305 		try {
306 			in = new ByteArrayInputStream(binaryJarBytes);
307 			jar = new Jar(null, in);
308 			List<String> packages = jar.getPackages();
309 
310 			out = new ByteArrayOutputStream();
311 			sourcesProvider.writeSources(packages, new ZipOutputStream(out));
312 
313 			IOUtils.closeQuietly(in);
314 			in = new ByteArrayInputStream(out.toByteArray());
315 			byte[] sourcesJar = RepoUtils.packageAsPdeSource(in,
316 					new DefaultNameVersion(artifact.getArtifactId(), artifact.getVersion()));
317 			Artifact sourcesArtifact = new DefaultArtifact(artifact.getGroupId(), artifact.getArtifactId() + ".source",
318 					"jar", artifact.getVersion());
319 			Node sourcesJarNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), sourcesArtifact, sourcesJar);
320 			sourcesJarNode.getSession().save();
321 
322 			if (log.isDebugEnabled())
323 				log.debug("Added sources " + sourcesArtifact + " for bundle " + artifact + "from source provider "
324 						+ sourcesProvider);
325 		} catch (Exception e) {
326 			throw new SlcException("Cannot get sources for " + artifact, e);
327 		} finally {
328 			IOUtils.closeQuietly(in);
329 			IOUtils.closeQuietly(out);
330 			if (jar != null)
331 				jar.close();
332 		}
333 	}
334 
335 	protected Artifact importZipEntry(Session javaSession, ZipEntry zentry, byte[] binaryJarBytes, String groupId)
336 			throws RepositoryException {
337 		ByteArrayInputStream in = null;
338 		Node newJarNode;
339 		try {
340 			in = new ByteArrayInputStream(binaryJarBytes);
341 			NameVersion nameVersion = RepoUtils.readNameVersion(in);
342 			if (nameVersion == null) {
343 				log.warn("Cannot identify " + zentry.getName());
344 				return null;
345 			}
346 			Artifact artifact = new DefaultArtifact(groupId, nameVersion.getName(), "jar", nameVersion.getVersion());
347 			newJarNode = RepoUtils.copyBytesAsArtifact(javaSession.getRootNode(), artifact, binaryJarBytes);
348 			osgiFactory.indexNode(newJarNode);
349 			newJarNode.getSession().save();
350 			if (log.isDebugEnabled()) {
351 				log.debug(zentry.getName() + " => " + artifact);
352 			}
353 
354 			if (sourcesProvider != null)
355 				addSource(javaSession, artifact, binaryJarBytes);
356 
357 			return artifact;
358 		} finally {
359 			IOUtils.closeQuietly(in);
360 		}
361 	}
362 
363 	private void addArtifactToIndex(Map<String, Set<Artifact>> index, String groupId, Artifact artifact) {
364 		if (!index.containsKey(groupId))
365 			index.put(groupId, new TreeSet<Artifact>(new ArtifactIdComparator()));
366 		index.get(groupId).add(artifact);
367 	}
368 
369 	public void setUri(String uri) {
370 		this.uri = uri;
371 	}
372 
373 	public void setWrappers(Map<String, BndWrapper> wrappers) {
374 		this.wrappers = wrappers;
375 	}
376 
377 	public void setOsgiFactory(OsgiFactory osgiFactory) {
378 		this.osgiFactory = osgiFactory;
379 	}
380 
381 	public void setVersion(String version) {
382 		this.version = version;
383 	}
384 
385 	public void setLicense(License license) {
386 		this.license = license;
387 	}
388 
389 	public void setPathMatcher(PathMatcher pathMatcher) {
390 		this.pathMatcher = pathMatcher;
391 	}
392 
393 	public void setIncludes(Map<String, String> includes) {
394 		this.includes = includes;
395 	}
396 
397 	public void setExcludes(List<String> excludes) {
398 		this.excludes = excludes;
399 	}
400 
401 	public void setMavenGroupIndexes(Boolean mavenGroupIndexes) {
402 		this.mavenGroupIndexes = mavenGroupIndexes;
403 	}
404 
405 	public void setSourcesProvider(SourcesProvider sourcesProvider) {
406 		this.sourcesProvider = sourcesProvider;
407 	}
408 
409 }