Skip to content

avoid marking non-existent project as indexed in the webapp #4800

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 16, 2025
Merged

avoid marking non-existent project as indexed in the webapp #4800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0e1f7a3
avoid marking project as indexed in the webapp if not known
vladak Jun 12, 2025
a659ca3
fix style and unused imports
vladak Jun 12, 2025
aa69d0e
use Optional to handle non-existent project
vladak Jun 13, 2025
667b442
rename to be more readable
vladak Jun 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 7 additions & 36 deletions opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
Expand All @@ -54,16 +55,12 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.zip.GZIPOutputStream;

import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.Entity;
import jakarta.ws.rs.core.Response;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools;
Expand Down Expand Up @@ -124,9 +121,6 @@
import org.opengrok.indexer.util.TandemPath;
import org.opengrok.indexer.web.Util;

import static org.opengrok.indexer.index.IndexerUtil.getWebAppHeaders;
import static org.opengrok.indexer.web.ApiUtils.waitForAsyncApi;

/**
* This class is used to create / update the index databases. Currently, we use
* one index database per project.
Expand Down Expand Up @@ -423,44 +417,21 @@ private void markProjectIndexed(Project project) {
return;
}

// Also need to store the correct value in configuration
// when indexer writes it to a file.
// Also need to store the correct value in configuration when indexer writes it to a file.
project.setIndexed(true);

if (env.getConfigURI() == null) {
return;
}

Response response;
try {
response = ClientBuilder.newBuilder().connectTimeout(env.getConnectTimeout(), TimeUnit.SECONDS).build()
.target(env.getConfigURI())
.path("api")
.path("v1")
.path("projects")
.path(Util.uriEncode(project.getName()))
.path("indexed")
.request()
.headers(getWebAppHeaders())
.put(Entity.text(""));
} catch (RuntimeException e) {
LOGGER.log(Level.WARNING, String.format("Could not notify the webapp that project %s was indexed",
project), e);
// If this project is not known to the webapp yet, there is no point in setting its indexed property.
Collection<String> webappProjects = IndexerUtil.getProjects(env.getConfigURI());
if (!webappProjects.contains(project.getName())) {
LOGGER.log(Level.FINEST, "Project {0} is not known to the webapp", project);
return;
}

if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
try {
response = waitForAsyncApi(response);
} catch (InterruptedException e) {
LOGGER.log(Level.WARNING, "interrupted while waiting for API response", e);
}
}

if (response.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
LOGGER.log(Level.WARNING, "Could not notify the webapp that project {0} was indexed: {1}",
new Object[] {project, response});
}
IndexerUtil.markProjectIndexed(env.getConfigURI(), project);
}

private static List<Repository> getRepositoriesForProject(Project project) {
Expand Down
77 changes: 72 additions & 5 deletions opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexerUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,14 @@
*/

/*
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
*/
package org.opengrok.indexer.index;

import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.core.GenericType;
import jakarta.ws.rs.core.MediaType;
import org.opengrok.indexer.configuration.Project;
import org.opengrok.indexer.configuration.RuntimeEnvironment;

import jakarta.ws.rs.ProcessingException;
Expand All @@ -35,11 +38,21 @@
import jakarta.ws.rs.core.MultivaluedHashMap;
import jakarta.ws.rs.core.MultivaluedMap;
import jakarta.ws.rs.core.Response;
import org.opengrok.indexer.logger.LoggerFactory;
import org.opengrok.indexer.web.Util;

import java.util.Collection;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.opengrok.indexer.web.ApiUtils.waitForAsyncApi;

public class IndexerUtil {

private static final Logger LOGGER = LoggerFactory.getLogger(IndexerUtil.class);

private IndexerUtil() {
}

Expand All @@ -57,24 +70,24 @@ public static MultivaluedMap<String, Object> getWebAppHeaders() {
}

/**
* Enable projects in the remote host application.
* Enable projects in the remote application.
* <p>
* NOTE: performs a check if the projects are already enabled,
* before making the change request
*
* @param host the url to the remote host
* @param webappUri the url to the remote web application
* @throws ResponseProcessingException in case processing of a received HTTP response fails
* @throws ProcessingException in case the request processing or subsequent I/O operation fails
* @throws WebApplicationException in case the response status code of the response returned by the server is not successful
*/
public static void enableProjects(final String host) throws
public static void enableProjects(final String webappUri) throws
ResponseProcessingException,
ProcessingException,
WebApplicationException {

try (Client client = ClientBuilder.newBuilder().
connectTimeout(RuntimeEnvironment.getInstance().getConnectTimeout(), TimeUnit.SECONDS).build()) {
final Invocation.Builder request = client.target(host)
final Invocation.Builder request = client.target(webappUri)
.path("api")
.path("v1")
.path("configuration")
Expand All @@ -92,4 +105,58 @@ public static void enableProjects(final String host) throws
}
}
}

/**
* Mark project as indexed via API call. Assumes the project is already known to the webapp.
* @param webappUri URI for the webapp
* @param project project to mark as indexed
*/
public static void markProjectIndexed(String webappUri, Project project) {
Response response;
try (Client client = ClientBuilder.newBuilder().
connectTimeout(RuntimeEnvironment.getInstance().getConnectTimeout(), TimeUnit.SECONDS).build()) {
response = client.target(webappUri)
.path("api")
.path("v1")
.path("projects")
.path(Util.uriEncode(project.getName()))
.path("indexed")
.request()
.headers(getWebAppHeaders())
.put(Entity.text(""));

if (response.getStatus() == Response.Status.ACCEPTED.getStatusCode()) {
try {
response = waitForAsyncApi(response);
} catch (InterruptedException e) {
LOGGER.log(Level.WARNING, "interrupted while waiting for API response", e);
}
}

if (response.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
LOGGER.log(Level.WARNING, "Could not notify the webapp that project {0} was indexed: {1}",
new Object[] {project, response});
}
} catch (RuntimeException e) {
LOGGER.log(Level.WARNING, String.format("Could not notify the webapp that project %s was indexed",
project), e);
}
}

/**
* @param webappUri URI for the webapp
* @return list of projects known to the webapp
*/
public static Collection<String> getProjects(String webappUri) {
try (Client client = ClientBuilder.newBuilder().
connectTimeout(RuntimeEnvironment.getInstance().getConnectTimeout(), TimeUnit.SECONDS).build()) {
final Invocation.Builder request = client.target(webappUri)
.path("api")
.path("v1")
.path("projects")
.request(MediaType.APPLICATION_JSON)
.headers(getWebAppHeaders());
return request.get(new GenericType<List<String>>() { } );
}
}
}
44 changes: 16 additions & 28 deletions opengrok-web/src/main/java/org/opengrok/web/api/v1/controller/ProjectsController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
Expand Down Expand Up @@ -135,17 +134,17 @@ private void addProjectWorkHorse(String projectName) {
// This is the goal of this action: if an existing project
// is re-added, this means its list of repositories has changed.
List<RepositoryInfo> repos = getRepositoriesInDir(projDir);
List<RepositoryInfo> allrepos = env.getRepositories();
synchronized (allrepos) {
List<RepositoryInfo> allRepositories = env.getRepositories();
synchronized (allRepositories) {
// newly added repository
repos.stream()
.filter(repo -> !allrepos.contains(repo))
.forEach(allrepos::add);
.filter(repo -> !allRepositories.contains(repo))
.forEach(allRepositories::add);
// deleted repository
Optional.ofNullable(map.get(project))
.stream().flatMap(Collection::stream)
.filter(repo -> !repos.contains(repo))
.forEach(allrepos::remove);
.forEach(allRepositories::remove);
}

map.put(project, repos);
Expand All @@ -163,10 +162,8 @@ private List<RepositoryInfo> getRepositoriesInDir(final File projDir) {
}

private Project disableProject(String projectName) {
Project project = env.getProjects().get(projectName);
if (project == null) {
throw new IllegalStateException("cannot get project \"" + projectName + "\"");
}
Project project = Optional.ofNullable(env.getProjects().get(projectName)).
orElseThrow(() -> new NotFoundException("cannot get project \"" + projectName + "\""));

// Remove the project from searches so no one can trip over incomplete index data.
project.setIndexed(false);
Expand Down Expand Up @@ -288,12 +285,9 @@ public Response deleteAnnotationCache(@Context HttpServletRequest request,
private Project getProjectFromName(String projectNameParam) {
// Avoid classification as a taint bug.
final String projectName = Laundromat.launderInput(projectNameParam);
Project project = env.getProjects().get(projectName);
if (project == null) {
throw new IllegalStateException("cannot get project \"" + projectName + "\"");
}

return project;
return Optional.ofNullable(env.getProjects().get(projectName)).
orElseThrow(() -> new NotFoundException("cannot get project \"" + projectName + "\""));
}

@DELETE
Expand Down Expand Up @@ -340,11 +334,8 @@ public Response markIndexed(@Context HttpServletRequest request, @PathParam("pro
// Avoid classification as a taint bug.
final String projectName = Laundromat.launderInput(projectNameParam);

Project project = env.getProjects().get(projectName);
if (project == null) {
LOGGER.log(Level.WARNING, "cannot find project ''{0}'' to mark as indexed", projectName);
throw new NotFoundException(String.format("project '%s' does not exist", projectName));
}
Project project = Optional.ofNullable(env.getProjects().get(projectName)).
orElseThrow(() -> new NotFoundException("cannot get project \"" + projectName + "\""));

project.setIndexed(true);

Expand All @@ -358,7 +349,7 @@ public Response markIndexed(@Context HttpServletRequest request, @PathParam("pro
Repository repo = getRepository(ri, CommandTimeoutType.RESTFUL);

if (repo != null && repo.getCurrentVersion() != null &&
repo.getCurrentVersion().length() > 0) {
!repo.getCurrentVersion().isEmpty()) {
// getRepository() always creates fresh instance
// of the Repository object so there is no need
// to call setCurrentVersion() on it.
Expand Down Expand Up @@ -412,17 +403,14 @@ public void set(
@GET
@Path("/{project}/property/{field}")
@Produces(MediaType.APPLICATION_JSON)
public Object get(@PathParam("project") String projectName, @PathParam("field") String field)
public Object get(@PathParam("project") String projectNameParam, @PathParam("field") String field)
throws IOException {
// Avoid classification as a taint bug.
projectName = Laundromat.launderInput(projectName);
final String projectName = Laundromat.launderInput(projectNameParam);
field = Laundromat.launderInput(field);

Project project = env.getProjects().get(projectName);
if (project == null) {
throw new WebApplicationException(
"cannot find project '" + projectName + "' to get a property", Response.Status.BAD_REQUEST);
}
Project project = Optional.ofNullable(env.getProjects().get(projectName)).
orElseThrow(() -> new NotFoundException("cannot get project \"" + projectName + "\""));
return ClassUtil.getFieldValue(project, field);
}

Expand Down
Loading