Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Backport] Simplify repository assembling and reference filtering #2851

Merged
merged 2 commits into from
Sep 24, 2023
Merged

[Backport] Simplify repository assembling and reference filtering #2851

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7124332
Minor code enhancements in p2-repository assembling code
HannesWell Aug 30, 2023
0e10008
Simplify logic of AssembleRepositoryMojo.repositoryReferenceFilter
HannesWell Sep 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions RELEASE_NOTES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ can now be filtered by their location using exclusion and inclusion patterns and
<repositoryReferenceFilter>
<exclude>
<location>https://foo.bar.org/hidden/**</location>
<location>https://foo.bar.org/secret/**</location>
<location>%regex[http(s)?:\/\/foo\.bar\.org\/secret\/.*]</location>
<location>![https://foo.bar.org/**]</location>
</exclude>
<include>%regex[http(s)?:\/\/foo\.bar\.org\/.*]</include>
</repositoryReferenceFilter>
</configuration>
</plugin>
Expand Down
7 changes: 2 additions & 5 deletions tycho-core/src/main/java/org/eclipse/tycho/core/TargetPlatformConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public enum InjectP2MavenMetadataHandling {

private Map<String, String> resolverProfileProperties = new HashMap<>();

List<Supplier<File>> lazyTargetFiles = new ArrayList<>();
private final List<Supplier<File>> lazyTargetFiles = new ArrayList<>();

private LocalArtifactHandling localArtifactHandling;

Expand Down Expand Up @@ -240,10 +240,7 @@ public void setFilters(List<TargetPlatformFilter> filters) {
}

public List<TargetPlatformFilter> getFilters() {
if (filters == null)
return Collections.emptyList();
else
return filters;
return filters == null ? Collections.emptyList() : filters;
}

public DependencyResolverConfiguration getDependencyResolverConfiguration() {
Expand Down
12 changes: 9 additions & 3 deletions tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

public class DestinationRepositoryDescriptor {

final File location;
final String name;
private final File location;
private final String name;
private final boolean compress;
private final boolean xzCompress;
private final boolean keepNonXzIndexFiles;
Expand All @@ -43,8 +43,14 @@ public DestinationRepositoryDescriptor(File location, String name, boolean compr
this.repositoryReferences = repositoryReferences;
}

public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress,
boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append) {
this(location, name, compress, xzCompress, keepNonXzIndexFiles, metaDataOnly, append, Collections.emptyMap(),
Collections.emptyList());
}

public DestinationRepositoryDescriptor(File location, String name) {
this(location, name, true, true, false, false, true, Collections.emptyMap(), Collections.emptyList());
this(location, name, true, true, false, false, true);
}

public File getLocation() {
Expand Down
5 changes: 2 additions & 3 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,7 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati

private static TychoMirrorApplication createMirrorApplication(RepositoryReferences sources,
DestinationRepositoryDescriptor destination, IProvisioningAgent agent) {
final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent,
destination.getExtraArtifactRepositoryProperties(), destination.getRepositoryReferences());
final TychoMirrorApplication mirrorApp = new TychoMirrorApplication(agent, destination);
mirrorApp.setRaw(false);

List<RepositoryDescriptor> sourceDescriptors = createSourceDescriptors(sources);
Expand Down Expand Up @@ -438,7 +437,7 @@ public void addMavenMappingRules(File repository, URI[] mavenRepositories) throw
repo.save();
DestinationRepositoryDescriptor desc = new DestinationRepositoryDescriptor(repository, repo.getName(),
new File(repository, "artifacts.xml.xz").exists(), new File(repository, "artifacts.xml.xz").exists(),
true, false, false, Collections.emptyMap(), Collections.emptyList());
true, false, false);
xzCompress(desc);
}
}
166 changes: 84 additions & 82 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
*******************************************************************************/
package org.eclipse.tycho.p2tools;

import static java.util.stream.Collectors.groupingBy;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
Expand All @@ -20,7 +22,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

Expand All @@ -41,12 +42,13 @@
import org.eclipse.equinox.p2.query.IQueryable;
import org.eclipse.equinox.p2.query.QueryUtil;
import org.eclipse.equinox.p2.repository.IRepository;
import org.eclipse.equinox.p2.repository.IRepositoryReference;
import org.eclipse.equinox.p2.repository.IRepositoryManager;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
import org.eclipse.equinox.p2.repository.metadata.IMetadataRepositoryManager;
import org.eclipse.tycho.TargetPlatform;
import org.eclipse.tycho.p2.tools.DestinationRepositoryDescriptor;
import org.eclipse.tycho.p2.tools.RepositoryReference;

public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfromp2.MirrorApplication {
Expand All @@ -58,14 +60,13 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom
private boolean includeAllSource;
private boolean includeRequiredBundles;
private boolean includeRequiredFeatures;
private TargetPlatform targetPlatform;
private boolean filterProvided;
private TargetPlatform targetPlatform;

public TychoMirrorApplication(IProvisioningAgent agent, Map<String, String> extraArtifactRepositoryProperties,
List<RepositoryReference> repositoryReferences) {
public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) {
super(agent);
this.extraArtifactRepositoryProperties = extraArtifactRepositoryProperties;
this.repositoryReferences = repositoryReferences;
this.extraArtifactRepositoryProperties = destination.getExtraArtifactRepositoryProperties();
this.repositoryReferences = destination.getRepositoryReferences();
this.removeAddedRepositories = false;
}

Expand All @@ -75,8 +76,7 @@ protected IArtifactRepository initializeDestination(RepositoryDescriptor toInit,
IArtifactRepository result = super.initializeDestination(toInit, mgr);
// simple.SimpleArtifactRepository.PUBLISH_PACK_FILES_AS_SIBLINGS is not public
result.setProperty("publishPackFilesAsSiblings", "true");
extraArtifactRepositoryProperties.entrySet()
.forEach(entry -> result.setProperty(entry.getKey(), entry.getValue()));
extraArtifactRepositoryProperties.forEach(result::setProperty);
return result;
}

Expand All @@ -85,38 +85,25 @@ protected Slicer createSlicer(SlicingOptions options) {
Map<String, String> context = options.getFilter();
boolean includeOptionalDependencies = options.includeOptionalDependencies();
boolean onlyFilteredRequirements = options.followOnlyFilteredRequirements();
boolean considerFilter = (context != null && context.size() > 1) ? true : false;
boolean considerFilter = context != null && context.size() > 1;
IMetadataRepository repository = getCompositeMetadataRepository();
return new PermissiveSlicer(repository, context, includeOptionalDependencies, options.isEverythingGreedy(),
options.forceFilterTo(), options.considerStrictDependencyOnly(), onlyFilteredRequirements) {
@Override
protected boolean isApplicable(IInstallableUnit iu, IRequirement req) {
if ((includeRequiredBundles || includeRequiredFeatures) && QueryUtil.isGroup(iu)) {
if (req instanceof IRequiredCapability capability) {
if (IInstallableUnit.NAMESPACE_IU_ID.equals(capability.getNamespace())) {
boolean isFeature = capability.getName().endsWith(FEATURE_GROUP);
if ((isFeature && includeRequiredFeatures) || (!isFeature && includeRequiredBundles)) {
if (!includeOptionalDependencies) {
if (req.getMin() == 0) {
return false;
}
}
IMatchExpression<IInstallableUnit> filter = req.getFilter();
if (considerFilter) {
if (onlyFilteredRequirements && filter == null) {
return false;
}
boolean filterMatches = filter == null || filter.isMatch(selectionContext);
if (filterMatches) {
}
return filterMatches;
}
if (filter == null && onlyFilteredRequirements) {
return false;
}
return true;
}
if ((includeRequiredBundles || includeRequiredFeatures) && QueryUtil.isGroup(iu)
&& req instanceof IRequiredCapability capability
&& IInstallableUnit.NAMESPACE_IU_ID.equals(capability.getNamespace())) {
boolean isFeature = capability.getName().endsWith(FEATURE_GROUP);
if ((isFeature && includeRequiredFeatures) || (!isFeature && includeRequiredBundles)) {
if (!includeOptionalDependencies && req.getMin() == 0) {
return false;
}
IMatchExpression<IInstallableUnit> filter = req.getFilter();
if (onlyFilteredRequirements && filter == null) {
return false;
}
return !considerFilter || filter == null || filter.isMatch(selectionContext);
}
}
return super.isApplicable(req);
Expand All @@ -133,11 +120,10 @@ public IQueryable<IInstallableUnit> slice(IInstallableUnit[] ius, IProgressMonit
if (includeAllSource && targetPlatform != null) {
Set<IInstallableUnit> collected = slice.query(QueryUtil.ALL_UNITS, null).toSet();
Set<IInstallableUnit> result = new HashSet<>(collected);
IQueryResult<IInstallableUnit> query = targetPlatform.getMetadataRepository()
.query(QueryUtil.ALL_UNITS, null);
Map<String, List<IInstallableUnit>> sourceIus = StreamSupport.stream(query.spliterator(), false)
var allUnits = targetPlatform.getMetadataRepository().query(QueryUtil.ALL_UNITS, null);
Map<String, List<IInstallableUnit>> sourceIus = stream(allUnits)
.filter(iu -> iu.getId().endsWith(SOURCE_SUFFIX))
.collect(Collectors.groupingBy(IInstallableUnit::getId));
.collect(groupingBy(IInstallableUnit::getId));
for (IInstallableUnit iu : collected) {
String id = iu.getId();
String sourceId = id.endsWith(FEATURE_GROUP)
Expand All @@ -161,70 +147,86 @@ public IQueryable<IInstallableUnit> slice(IInstallableUnit[] ius, IProgressMonit
protected IMetadataRepository initializeDestination(RepositoryDescriptor toInit, IMetadataRepositoryManager mgr)
throws ProvisionException {
IMetadataRepository result = super.initializeDestination(toInit, mgr);
List<? extends IRepositoryReference> iRepoRefs = repositoryReferences.stream()
.flatMap(TychoMirrorApplication::toSpiRepositoryReferences).toList();
result.addReferences(iRepoRefs);
var refs = repositoryReferences.stream().flatMap(TychoMirrorApplication::toSpiRepositoryReferences).toList();
result.addReferences(refs);
return result;
}

private static Stream<org.eclipse.equinox.p2.repository.spi.RepositoryReference> toSpiRepositoryReferences(
RepositoryReference rr) {
return Stream.of(toSpiRepositoryReference(rr, IRepository.TYPE_METADATA),
toSpiRepositoryReference(rr, IRepository.TYPE_ARTIFACT));
}

private static org.eclipse.equinox.p2.repository.spi.RepositoryReference toSpiRepositoryReference(
RepositoryReference rr, int type) {
return new org.eclipse.equinox.p2.repository.spi.RepositoryReference(URI.create(rr.getLocation()), rr.getName(),
type, rr.isEnable() ? IRepository.ENABLED : IRepository.NONE);
return Stream.of(IRepository.TYPE_METADATA, IRepository.TYPE_ARTIFACT).map(type -> {
URI location = URI.create(rr.getLocation());
int options = rr.isEnable() ? IRepository.ENABLED : IRepository.NONE;
return new org.eclipse.equinox.p2.repository.spi.RepositoryReference(location, rr.getName(), type, options);
});
}

@Override
protected List<IArtifactKey> collectArtifactKeys(Collection<IInstallableUnit> ius, IProgressMonitor monitor)
throws ProvisionException {
List<IArtifactKey> keys = super.collectArtifactKeys(ius, monitor);
if (isFilterProvidedItems()) {
List<IArtifactRepository> referencedRepositories = new ArrayList<>();
for (RepositoryReference reference : repositoryReferences) {
String location = reference.getLocation();
try {
referencedRepositories
.add(getArtifactRepositoryManager().loadRepository(new URI(location), monitor));
} catch (URISyntaxException e) {
throw new ProvisionException("Can't parse referenced URI!", e);
}
}
keys.removeIf(key -> referencedRepositories.stream().anyMatch(repo -> repo.contains(key)));
removeProvidedItems(keys, getArtifactRepositoryManager(), monitor);
}
return keys;
}

private boolean isFilterProvidedItems() {
return filterProvided && !repositoryReferences.isEmpty();
}

@Override
protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)
throws ProvisionException {
Set<IInstallableUnit> units = super.collectUnits(slice, monitor);
if (isFilterProvidedItems()) {
List<IMetadataRepository> referencedRepositories = new ArrayList<>();
for (RepositoryReference reference : repositoryReferences) {
String location = reference.getLocation();
try {
referencedRepositories
.add(getMetadataRepositoryManager().loadRepository(new URI(location), monitor));
} catch (URISyntaxException e) {
throw new ProvisionException("Can't parse referenced URI!", e);
}
}
units.removeIf(unit -> referencedRepositories.stream().anyMatch(repo -> {
return !repo.query(QueryUtil.createIUQuery(unit.getId(), unit.getVersion()), monitor).isEmpty();
}));
removeProvidedItems(units, getMetadataRepositoryManager(), monitor);
}
return units;
}

private boolean isFilterProvidedItems() {
return filterProvided && !repositoryReferences.isEmpty();
}

private <T> void removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
IProgressMonitor monitor) throws ProvisionException {
List<IRepository<T>> referencedRepositories = new ArrayList<>();
for (RepositoryReference reference : repositoryReferences) {
try {
URI location = new URI(reference.getLocation());
IRepository<T> repository = loadRepository(repoManager, location, monitor);
referencedRepositories.add(repository);
} catch (URISyntaxException e) {
throw new ProvisionException("Can't parse referenced URI!", e);
}
}
allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo, e)));
}

//TODO: just call IRepositoryManager.loadRepository() once available: https://github.com/eclipse-equinox/p2/pull/311
@SuppressWarnings("unchecked")
private static <T> IRepository<T> loadRepository(IRepositoryManager<T> mgr, URI location, IProgressMonitor monitor)
throws ProvisionException {
if (mgr instanceof IArtifactRepositoryManager artifactRepoManager) {
return (IRepository<T>) artifactRepoManager.loadRepository(location, monitor);
} else if (mgr instanceof IMetadataRepositoryManager metadataMangager) {
return (IRepository<T>) metadataMangager.loadRepository(location, monitor);
}
throw new AssertionError("Unsupported IRepositoryManager type" + mgr.getClass());
}

//TODO: just call IRepositoryManager.contains() once available: https://github.com/eclipse-equinox/p2/pull/314
private static <T> boolean contains(IRepository<T> repository, T element) {
if (repository instanceof IArtifactRepository artifactRepository) {
return artifactRepository.contains((IArtifactKey) element);
} else if (repository instanceof IMetadataRepository metadataRepository) {
return !metadataRepository.query(QueryUtil.createIUQuery((IInstallableUnit) element), null).isEmpty();
}
throw new AssertionError("Unsupported IRepository type" + repository.getClass());
}

//TODO: use query.stream() once available: https://github.com/eclipse-equinox/p2/pull/312 is available
private static <T> Stream<T> stream(IQueryResult<T> result) {
return StreamSupport.stream(result.spliterator(), false);
}

public void setIncludeSources(boolean includeAllSource, TargetPlatform targetPlatform) {
this.includeAllSource = includeAllSource;
this.targetPlatform = targetPlatform;
Expand All @@ -234,12 +236,12 @@ public void setIncludeRequiredBundles(boolean includeRequiredBundles) {
this.includeRequiredBundles = includeRequiredBundles;
}

public void setFilterProvided(boolean filterProvided) {
this.filterProvided = filterProvided;
}

public void setIncludeRequiredFeatures(boolean includeRequiredFeatures) {
this.includeRequiredFeatures = includeRequiredFeatures;
}

public void setFilterProvided(boolean filterProvided) {
this.filterProvided = filterProvided;
}

}
3 changes: 1 addition & 2 deletions .../tycho-p2-extras-plugin/src/main/java/org/eclipse/tycho/plugins/p2/extras/MirrorMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,8 +246,7 @@ public void execute() throws MojoExecutionException, MojoFailureException {
name = "";
}
final DestinationRepositoryDescriptor destinationDescriptor = new DestinationRepositoryDescriptor(destination,
name, compress, xzCompress, keepNonXzIndexFiles, mirrorMetadataOnly, append, Collections.emptyMap(),
Collections.emptyList());
name, compress, xzCompress, keepNonXzIndexFiles, mirrorMetadataOnly, append);
getLog().info("Mirroring to " + destination);
try {
mirrorService.mirrorStandalone(sourceDescriptor, destinationDescriptor, createIUDescriptions(),
Expand Down
2 changes: 1 addition & 1 deletion tycho-its/projects/p2Repository.repositoryRef.filter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@
<exclude>
<location>https://download.eclipse.org/lsp4e/**</location>
<location>https://download.eclipse.org/lsp4j/**</location>
<location>![%regex[http(s)?:\/\/download\.eclipse\.org\/.*]]</location>
</exclude>
<include>%regex[http(s)?:\/\/download\.eclipse\.org\/.*]</include>
</repositoryReferenceFilter>
</configuration>
</plugin>
Expand Down
Loading
Loading