How Semantic Search Works

public class SearchComparison {
  public static void main(String[] args) {
    if (args.length != 1) {
      System.out.println("Usage: SearchComparison <query>");
      System.exit(1);
    }
    var searchers = List.of(new LuceneTextSearch(), new SemanticTextSearch());

    var candidates =
        List.of(
            "What is the best time to plant rice?",
            "How often should I water tomato plants?",
            "What fertilizer is good for wheat crops?",
            "How can I control pests on my cotton farm?",
            "Which crop is best for sandy soil?",
            "Is there a difference between four-wheel drive and all-wheel drive?");

    var query = args[0];
    System.out.printf("\nCandidates for query \"%s\"\n\n", query);
    for (TextSearch searcher : searchers) {
      System.out.printf("Searching with \"%s\"\n", searcher.getClass().getSimpleName());
      searcher.filterCandidates(candidates, query).forEach(System.out::println);
      System.out.println("---");
    }
  }
}

public class LuceneTextSearch implements TextSearch {
  private static final String FIELD_DESCRIPTION = "description";

  private static final Analyzer analyzer = new EnglishAnalyzer();
  private static final QueryParser parser = new QueryParser(FIELD_DESCRIPTION, analyzer);

  @Override
  public List<Result> filterCandidates(List<String> candidates, String query) {
    var indices = buildIndices(candidates);
    Query parsedQuery = getParsedQuery(query);
    return indices.entrySet().stream()
        .map(
            entry -> {
              var score = entry.getValue().search(parsedQuery);
              return new Result(entry.getKey(), score);
            })
        .sorted(Comparator.comparingDouble(Result::score).reversed())
        .toList();
  }

  private static Query getParsedQuery(String query) {
    Query parsedQuery = null;
    try {
      parsedQuery = parser.parse(query);
    } catch (ParseException e) {
      throw new RuntimeException(e);
    }
    return parsedQuery;
  }

  private Map<String, MemoryIndex> buildIndices(List<String> candidates) {
    return candidates.stream()
        .collect(
            Collectors.toMap(
                Function.identity(),
                candidate -> {
                  var idx = new MemoryIndex();
                  idx.addField(FIELD_DESCRIPTION, candidate, analyzer);
                  return idx;
                }));
  }
}

public class SemanticTextSearch implements TextSearch {
  private EmbeddingModel embeddingModel;

  public SemanticTextSearch() {
    getEmbeddingModel();
  }

  @Override
  public List<Result> filterCandidates(List<String> candidates, String query) {
    var embeddings = createEmbeddings(candidates);
    var queryEmbedding = getEmbeddingModel().embed(query).content();
    return embeddings.entrySet().stream()
        .map(
            entry -> {
              var score = CosineSimilarity.between(entry.getValue(), queryEmbedding);
              return new Result(entry.getKey(), score);
            })
        .sorted(Comparator.comparingDouble(Result::score).reversed())
        .toList();
  }

  private Map<String, Embedding> createEmbeddings(List<String> candidates) {
    return candidates.stream()
        .collect(
            Collectors.toMap(
                Function.identity(), candidate -> getEmbeddingModel().embed(candidate).content()));
  }

  private EmbeddingModel getEmbeddingModel() {
    if (embeddingModel == null) {
      embeddingModel = new BgeSmallEnV15EmbeddingModel();
    }
    return embeddingModel;
  }
}

Candidates for query "when is a good time to plant cotton?"

Searching with "LuceneTextSearch"
0.26 What is the best time to plant rice?
0.13 How often should I water tomato plants?
0.13 What fertilizer is good for wheat crops?
0.00 Which crop is best for sandy soil?
0.00 How can I control pests on my cotton farm?
0.00 Is there a difference between four-wheel drive and all-wheel drive?
---
Searching with "SemanticTextSearch"
0.77 What is the best time to plant rice?
0.69 How can I control pests on my cotton farm?
0.66 Which crop is best for sandy soil?
0.61 How often should I water tomato plants?
0.58 What fertilizer is good for wheat crops?
0.38 Is there a difference between four-wheel drive and all-wheel drive?
---

Candidates for query "We want to avoid insects"

Searching with "LuceneTextSearch"
0.00 Which crop is best for sandy soil?
0.00 How can I control pests on my cotton farm?
0.00 How often should I water tomato plants?
0.00 What fertilizer is good for wheat crops?
0.00 What is the best time to plant rice?
0.00 Is there a difference between four-wheel drive and all-wheel drive?
---
Searching with "SemanticTextSearch"
0.72 How can I control pests on my cotton farm?
0.54 What is the best time to plant rice?
0.54 Which crop is best for sandy soil?
0.50 What fertilizer is good for wheat crops?
0.50 How often should I water tomato plants?
0.41 Is there a difference between four-wheel drive and all-wheel drive?
---