diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexManager.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexManager.java index 4789d4f2b8c8..bbcb511d6b43 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexManager.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexManager.java @@ -328,7 +328,6 @@ boolean loadIndexingContext(final RepositoryInfo info) throws IOException { break LOAD; // XXX does it suffice to just return here, or is code after block needed? } } - LOGGER.log(Level.FINE, "Loading Context: {0}", info.getId()); List creators; if (info.isLocal()) { @@ -348,7 +347,13 @@ boolean loadIndexingContext(final RepositoryInfo info) throws IOException { new NotifyingIndexCreator() ); } + + if (info.isRemoteDownloadable() && !indexExists(getIndexDirectory(info))) { + tryMoveRemoteIndexFromOldCache(info); + } + try { + LOGGER.log(Level.FINE, "Loading Context: {0}", info.getId()); addIndexingContextForced(info, creators); LOGGER.log(Level.FINE, "using index creators: {0}", creators); } catch (IOException | IllegalArgumentException ex) { // IAE thrown by lucene on index version incompatibilites @@ -387,6 +392,62 @@ boolean loadIndexingContext(final RepositoryInfo info) throws IOException { } } + /// Moves the index from the previous cache folder to the current folder if: + /// - both cache folders share the same parent + /// - the cache folder is from an older release + /// - the index itself is still somewhat up to date + /// - current NB instance is not a dev build + private void tryMoveRemoteIndexFromOldCache(RepositoryInfo info) { + + String buildnumber = System.getProperty("netbeans.buildnumber"); + if (buildnumber == null) { + return; // tests + } + int ourVersion; + try { + String debugRelease = System.getProperty("maven.indexing.diag.release"); + ourVersion = debugRelease != null + ? Integer.parseInt(debugRelease) + : Integer.parseInt(buildnumber.split("-")[0]); + } catch (NumberFormatException ignore) { + return; + } + + Path cacheParent = Places.getCacheDirectory().toPath().getParent(); + if (cacheParent != null) { + try (Stream caches = Files.list(cacheParent)) { + record CacheFolder(Path path, int version) {} + + Optional oldCache = caches + .filter(path -> Files.exists(path.resolve(".lastUsedVersion"))) + .map(path -> { + try { + int version = Integer.parseInt(Files.readString(path.resolve(".lastUsedVersion"))); + return new CacheFolder(path, version); + } catch (IOException | NumberFormatException ex) { + return null; + } + }) + .filter(Objects::nonNull) + .filter(cache -> cache.version < ourVersion) + .max((c1, c2) -> c1.version - c2.version); + + if (oldCache.isPresent()) { + Path oldIndex = getIndexDirectory(oldCache.get().path, info); + if (indexExists(oldIndex)) { + Instant lastUse = Files.getLastModifiedTime(oldIndex.resolve("timestamp")).toInstant(); + if (Instant.now().minus(30, ChronoUnit.DAYS).isBefore(lastUse)) { + LOGGER.log(Level.INFO, "moving index from old cache [{0}]", oldIndex); + Files.move(oldIndex, getIndexDirectory(info)); + } + } + } + } catch (Exception ex) { + LOGGER.log(Level.WARNING, "index import failed: {0}", ex.getMessage()); + } + } + } + //always call from mutex.writeAccess private void unloadIndexingContext(final String repo) throws IOException { assert getRepoMutex(repo).isWriteAccess(); @@ -415,9 +476,8 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr indexingMutexes.add(mutex); } boolean fetchFailed = false; + boolean fullUpdate = false; long t = System.currentTimeMillis(); - IndexUpdateResult fetchUpdateResult = null; - RemoteIndexTransferListener listener = null; try { IndexingContext indexingContext = getIndexingContexts().get(repo.getId()); if (indexingContext == null) { @@ -426,8 +486,7 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr } if (repo.isRemoteDownloadable()) { LOGGER.log(Level.FINE, "Indexing Remote Repository: {0}", repo.getId()); - listener = new RemoteIndexTransferListener(repo); - try { + try (RemoteIndexTransferListener listener = new RemoteIndexTransferListener(repo)) { String protocol = URI.create(indexingContext.getIndexUpdateUrl()).getScheme(); SettingsDecryptionResult settings = embedder.lookup(SettingsDecrypter.class).decrypt(new DefaultSettingsDecryptionRequest(EmbedderFactory.getOnlineEmbedder().getSettings())); AuthenticationInfo wagonAuth = null; @@ -496,16 +555,40 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr } try { removeGroupCache(repo); - fetchUpdateResult = remoteIndexUpdater.fetchAndUpdateIndex(iur); + IndexUpdateResult result = remoteIndexUpdater.fetchAndUpdateIndex(iur); + fullUpdate = result.isFullUpdate(); storeGroupCache(repo, indexingContext); // register indexed repo in services view - if (fetchUpdateResult.isFullUpdate() && fetchUpdateResult.isSuccessful()) { + if (result.isFullUpdate() && result.isSuccessful()) { RepositoryPreferences.getInstance().addOrModifyRepositoryInfo(repo); } } catch (IOException | AlreadyClosedException | IllegalArgumentException ex) { // AlreadyClosedException can happen in low storage situations when lucene is trying to handle IOEs // IllegalArgumentException signals remote archive format problems fetchFailed = true; + + Path tmpFolder = Path.of(System.getProperty("java.io.tmpdir")); + Path cacheFolder = getIndexDirectory(); + + long freeTmpSpace = getFreeSpaceInMB(tmpFolder); + long freeCacheSpace = getFreeSpaceInMB(cacheFolder); + + if (isNoSpaceLeftOnDevice(ex) || freeCacheSpace < 1000 || freeTmpSpace < 1000) { + + long downloaded = listener.getUnits() * 1024; + LOGGER.log(Level.INFO, "Downloaded maven index file has size {0} (zipped). The usable space in [cache]:{1} is {2} MB and in [tmp]:{3} is {4} MB.", + new Object[] {downloaded, cacheFolder, freeCacheSpace, tmpFolder, freeTmpSpace}); + LOGGER.log(Level.WARNING, "Download/Extraction failed due to low storage, indexing is now disabled.", ex); + + // disable indexing and tell user about it + RepositoryPreferences.setIndexRepositories(false); + + IndexingNotificationProvider np = Lookup.getDefault().lookup(IndexingNotificationProvider.class); + if(np != null) { + np.notifyError(Bundle.MSG_NoSpace(repo.getName(), "[cache]:"+cacheFolder.toString(), freeCacheSpace, "[tmp]:"+tmpFolder.toString(), freeTmpSpace)); + } + unloadIndexingContext(repo.getId()); + } throw new IOException("Failed to load maven-index for: " + indexingContext.getRepositoryUrl(), ex); } catch (RuntimeException ex) { // thread pools, like the one used in maven-indexer's IndexDataReader, may suppress cancellation exceptions @@ -529,10 +612,8 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr LOGGER.log(Level.WARNING, "cleanup failed"); } } - } finally { - listener.close(); } - } else { + } else if (repo.isLocal()) { LOGGER.log(Level.FINE, "Indexing Local Repository: {0}", repo.getId()); if (!indexingContext.getRepository().exists()) { //#210743 @@ -553,28 +634,6 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr if(e.getCause() instanceof ResourceDoesNotExistException) { fireChange(repo, () -> repo.fireNoIndex()); } - Path tmpFolder = Path.of(System.getProperty("java.io.tmpdir")); - Path cacheFolder = getIndexDirectory(); - - long freeTmpSpace = getFreeSpaceInMB(tmpFolder); - long freeCacheSpace = getFreeSpaceInMB(cacheFolder); - - if (isNoSpaceLeftOnDevice(e) || freeCacheSpace < 1000 || freeTmpSpace < 1000) { - - long downloaded = listener != null ? listener.getUnits() * 1024 : -1; - LOGGER.log(Level.INFO, "Downloaded maven index file has size {0} (zipped). The usable space in [cache]:{1} is {2} MB and in [tmp]:{3} is {4} MB.", - new Object[] {downloaded, cacheFolder, freeCacheSpace, tmpFolder, freeTmpSpace}); - LOGGER.log(Level.WARNING, "Download/Extraction failed due to low storage, indexing is now disabled.", e); - - // disable indexing and tell user about it - RepositoryPreferences.setIndexRepositories(false); - - IndexingNotificationProvider np = Lookup.getDefault().lookup(IndexingNotificationProvider.class); - if(np != null) { - np.notifyError(Bundle.MSG_NoSpace(repo.getName(), "[cache]:"+cacheFolder.toString(), freeCacheSpace, "[tmp]:"+tmpFolder.toString(), freeTmpSpace)); - } - unloadIndexingContext(repo.getId()); - } throw e; } catch (Cancellation x) { pauseRemoteRepoIndexing(120); // pause a while @@ -583,12 +642,12 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr throw new IOException("could not find protocol handler for " + repo.getRepositoryUrl(), x); } finally { String kind; - if (fetchUpdateResult != null) { - kind = fetchUpdateResult.isFullUpdate() ? "download, create" : "incremental download, update"; + if (repo.isRemoteDownloadable()) { + kind = fullUpdate ? "download, create" : "incremental download, update"; } else { - kind = "scan"; + kind = repo.isLocal() ? "scan" : "none"; } - LOGGER.log(Level.INFO, "Indexing [{0}] of {1} took {2}s.", new Object[]{kind, repo.getId(), String.format("%.2f", (System.currentTimeMillis() - t)/1000.0f)}); + LOGGER.log(Level.INFO, "Indexing [{0}] of {1} took {2}s.", new Object[]{kind, repo.getId(), "%.2f".formatted((System.currentTimeMillis() - t)/1000.0f)}); synchronized (indexingMutexes) { indexingMutexes.remove(mutex); } @@ -936,6 +995,10 @@ static Path getIndexDirectory(final RepositoryInfo info) { return getIndexDirectory().resolve(info.getId()); } + private static Path getIndexDirectory(Path cache, RepositoryInfo info) { + return cache.resolve("mavenindex").resolve(info.getId()); + } + private static Path getAllGroupCacheFile(final RepositoryInfo info) { return getIndexDirectory(info).resolve(GROUP_CACHE_ALL_PREFIX + "." + GROUP_CACHE_ALL_SUFFIX); } diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java index 97690f8fd30a..72c34540eed1 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/RemoteIndexTransferListener.java @@ -38,7 +38,7 @@ import org.openide.util.Cancellable; import org.openide.util.NbBundle.Messages; -public class RemoteIndexTransferListener implements TransferListener, Cancellable { +public class RemoteIndexTransferListener implements TransferListener, Cancellable, AutoCloseable { private static final Logger LOG = Logger.getLogger(RemoteIndexTransferListener.class.getName()); @@ -167,7 +167,8 @@ void unpackingProgress(String label) { handle.progress(label); } - void close() { + @Override + public void close() { handle.finish(); }