findByKind(String kind);
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/ERecognitionProviderKind.java b/src/main/java/org/icroco/pholio/infra/recognition/ERecognitionProviderKind.java
new file mode 100644
index 0000000..1b25527
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/ERecognitionProviderKind.java
@@ -0,0 +1,18 @@
+package org.icroco.pholio.infra.recognition;
+
+/**
+ * Which wire contract a {@link RecognitionProviderConfig} speaks. Unlike {@code EGeocodingProviderKind},
+ * there is no well-known third-party face/animal-recognition API to standardise on, so — for now — a remote
+ * recognition provider is expected to be a small HTTP service speaking Pholio's own contract: {@code POST}
+ * to {@link RecognitionProviderConfig#urlTemplate()} a JSON body {@code {"apiKey", "width", "height",
+ * "imageBase64"}} (the decoded image, JPEG-encoded, then base64), and reply with a JSON array of
+ * {@code {"kind":"PERSON"|"ANIMAL","x","y","w","h","confidence","label"}} (the same normalized {@code
+ * stArea} center/size convention {@link org.icroco.pholio.domain.recognition.BoundingBox} uses).
+ *
+ * Kept as a real enum rather than inlining {@link #HTTP_JSON} everywhere so a second, differently-shaped
+ * remote contract can be added later as one more case, the same reason {@code EGeocodingProviderKind} does.
+ */
+public enum ERecognitionProviderKind {
+
+ HTTP_JSON
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/EmbeddingCodec.java b/src/main/java/org/icroco/pholio/infra/recognition/EmbeddingCodec.java
new file mode 100644
index 0000000..181781d
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/EmbeddingCodec.java
@@ -0,0 +1,40 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.jspecify.annotations.Nullable;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+/**
+ * {@code float[]} embedding vector <-> the little-endian {@code byte[]} stored in
+ * {@code media_face_region.embedding} — H2/JDBC has no native vector/array column type, and a plain byte
+ * buffer needs no third-party (de)serialization library for something this small (typically 128-512 floats).
+ */
+public final class EmbeddingCodec {
+
+ private EmbeddingCodec() {
+ }
+
+ public static byte @Nullable [] toBytes(float @Nullable [] embedding) {
+ if (embedding == null) {
+ return null;
+ }
+ ByteBuffer buffer = ByteBuffer.allocate(embedding.length * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ for (float value : embedding) {
+ buffer.putFloat(value);
+ }
+ return buffer.array();
+ }
+
+ public static float @Nullable [] fromBytes(byte @Nullable [] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
+ float[] embedding = new float[bytes.length / Float.BYTES];
+ for (int i = 0; i < embedding.length; i++) {
+ embedding[i] = buffer.getFloat();
+ }
+ return embedding;
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/FaceClusteringService.java b/src/main/java/org/icroco/pholio/infra/recognition/FaceClusteringService.java
new file mode 100644
index 0000000..32407da
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/FaceClusteringService.java
@@ -0,0 +1,171 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.recognition.EEntityKind;
+import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionEntity;
+import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionRepository;
+import org.icroco.pholio.infra.persistence.recognition.PersonEntity;
+import org.icroco.pholio.infra.persistence.recognition.PersonRepository;
+import org.icroco.pholio.infra.preferences.AppPreferences;
+import org.jspecify.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Groups unnamed {@link EEntityKind#PERSON} face regions into {@link PersonEntity} identities by embedding
+ * similarity, entirely automatically — no manual step, the "Person 1", "Person 2"... clusters the future
+ * person-management panel will let a user name. Animal regions are never clustered: individual animal
+ * identity isn't attempted in this iteration, only the species label already on the region.
+ *
+ *
Brute-force cosine similarity against one centroid per existing cluster, not a full pairwise
+ * comparison against every previously-seen face — acceptable at "one photo library" scale (thousands, not
+ * millions, of faces); see the implementation plan's own risk notes for when that would need revisiting.
+ */
+@Service
+public class FaceClusteringService {
+
+ private static final Logger log = LoggerFactory.getLogger(FaceClusteringService.class);
+
+ /** SFace's own calibrated same-identity cosine threshold (OpenCV Zoo's face_recognition_sface README). */
+ private static final double DEFAULT_THRESHOLD = 0.363;
+
+ private final MediaFaceRegionRepository regionRepository;
+ private final PersonRepository personRepository;
+ private final AppPreferences preferences;
+
+ public FaceClusteringService(MediaFaceRegionRepository regionRepository, PersonRepository personRepository,
+ AppPreferences preferences) {
+ this.regionRepository = regionRepository;
+ this.personRepository = personRepository;
+ this.preferences = preferences;
+ }
+
+ /** Links every unlinked {@link EEntityKind#PERSON} region to an existing or freshly-minted {@link PersonEntity}. */
+ public void clusterUnnamedPersons() {
+ List unassigned = regionRepository.findByKindAndPersonIdIsNull(EEntityKind.PERSON.name());
+ if (unassigned.isEmpty()) {
+ return;
+ }
+ double threshold = preferences.getValueOr("recognition", "person-cluster-threshold", Double.class, DEFAULT_THRESHOLD);
+
+ List clusters = loadExistingClusters();
+ List toSave = new ArrayList<>();
+ for (MediaFaceRegionEntity region : unassigned) {
+ float[] embedding = EmbeddingCodec.fromBytes(region.getEmbedding());
+ if (embedding == null) {
+ log.warn("Person region {} has no embedding, cannot cluster it", region.getId());
+ continue;
+ }
+ Cluster best = bestMatch(clusters, embedding, threshold);
+ if (best == null) {
+ best = new Cluster(createPerson(), embedding.clone(), 1);
+ clusters.add(best);
+ }
+ else {
+ best.accumulate(embedding);
+ }
+ region.setPersonId(best.personId());
+ toSave.add(region);
+ }
+ if (!toSave.isEmpty()) {
+ regionRepository.saveAll(toSave);
+ log.info("Clustered {} face region(s) into {} person(s)", toSave.size(), clusters.size());
+ }
+ }
+
+ private List loadExistingClusters() {
+ List clusters = new ArrayList<>();
+ for (PersonEntity person : personRepository.findByKind(EEntityKind.PERSON.name())) {
+ Long personId = Objects.requireNonNull(person.getId(), "A persisted PersonEntity always has an id");
+ List embeddings = regionRepository.findByPersonId(personId).stream()
+ .map(region -> EmbeddingCodec.fromBytes(region.getEmbedding()))
+ .filter(Objects::nonNull)
+ .toList();
+ if (!embeddings.isEmpty()) {
+ clusters.add(new Cluster(personId, mean(embeddings), embeddings.size()));
+ }
+ }
+ return clusters;
+ }
+
+ private static @Nullable Cluster bestMatch(List clusters, float[] embedding, double threshold) {
+ Cluster best = null;
+ double bestScore = threshold;
+ for (Cluster cluster : clusters) {
+ double score = cosineSimilarity(cluster.centroid(), embedding);
+ if (score >= bestScore) {
+ best = cluster;
+ bestScore = score;
+ }
+ }
+ return best;
+ }
+
+ private Long createPerson() {
+ PersonEntity saved = personRepository.save(PersonEntity.builder()
+ .kind(EEntityKind.PERSON.name())
+ .name(null)
+ .createdAt(Instant.now())
+ .build());
+ return Objects.requireNonNull(saved.getId(), "A freshly saved PersonEntity always has an id");
+ }
+
+ private static float[] mean(List embeddings) {
+ float[] mean = new float[embeddings.getFirst().length];
+ for (float[] embedding : embeddings) {
+ for (int i = 0; i < mean.length; i++) {
+ mean[i] += embedding[i];
+ }
+ }
+ for (int i = 0; i < mean.length; i++) {
+ mean[i] /= embeddings.size();
+ }
+ return mean;
+ }
+
+ private static double cosineSimilarity(float[] a, float[] b) {
+ double dot = 0, normA = 0, normB = 0;
+ for (int i = 0; i < a.length; i++) {
+ dot += a[i] * b[i];
+ normA += a[i] * a[i];
+ normB += b[i] * b[i];
+ }
+ if (normA == 0 || normB == 0) {
+ return 0;
+ }
+ return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+ }
+
+ /** A running centroid — its own field, mutated in place as more regions join it within one clustering pass. */
+ private static final class Cluster {
+ private final Long personId;
+ private final float[] centroid;
+ private int count;
+
+ private Cluster(Long personId, float[] centroid, int count) {
+ this.personId = personId;
+ this.centroid = centroid;
+ this.count = count;
+ }
+
+ private Long personId() {
+ return personId;
+ }
+
+ private float[] centroid() {
+ return centroid;
+ }
+
+ private void accumulate(float[] embedding) {
+ count++;
+ for (int i = 0; i < centroid.length; i++) {
+ centroid[i] += (embedding[i] - centroid[i]) / count;
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/FaceRegionQueryService.java b/src/main/java/org/icroco/pholio/infra/recognition/FaceRegionQueryService.java
new file mode 100644
index 0000000..7f791ea
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/FaceRegionQueryService.java
@@ -0,0 +1,100 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+import org.icroco.pholio.domain.recognition.EEntityKind;
+import org.icroco.pholio.domain.recognition.MediaFaceRegion;
+import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionEntity;
+import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionMapper;
+import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionRepository;
+import org.icroco.pholio.infra.persistence.recognition.PersonEntity;
+import org.icroco.pholio.infra.persistence.recognition.PersonRepository;
+import org.jspecify.annotations.Nullable;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Reads and replaces a {@code MediaFile}'s detected regions — used by {@link MediaRecognitionService} (writes,
+ * after each detection pass), {@code MediaInfoPane} (reads, for its persons/animals row) and the future
+ * person-management panel.
+ */
+@Service
+public class FaceRegionQueryService {
+
+ private final MediaFaceRegionRepository repository;
+ private final MediaFaceRegionMapper mapper;
+ private final PersonRepository personRepository;
+
+ public FaceRegionQueryService(MediaFaceRegionRepository repository, MediaFaceRegionMapper mapper,
+ PersonRepository personRepository) {
+ this.repository = repository;
+ this.mapper = mapper;
+ this.personRepository = personRepository;
+ }
+
+ public List findByMediaFile(Long mediaFileId) {
+ return repository.findByMediaFileId(mediaFileId).stream().map(mapper::toDomain).toList();
+ }
+
+ /** One entry per detected region, name resolved for {@link EEntityKind#PERSON} rows already linked to a named {@code Person}. */
+ public List findDisplayRegionsFor(Long mediaFileId) {
+ List regions = repository.findByMediaFileId(mediaFileId);
+ if (regions.isEmpty()) {
+ return List.of();
+ }
+ Set personIds = regions.stream().map(MediaFaceRegionEntity::getPersonId).filter(java.util.Objects::nonNull).collect(Collectors.toSet());
+ Map namesById = personIds.isEmpty() ? Map.of()
+ : java.util.stream.StreamSupport.stream(personRepository.findAllById(personIds).spliterator(), false)
+ .filter(person -> person.getName() != null)
+ .collect(Collectors.toMap(PersonEntity::getId, PersonEntity::getName));
+ return regions.stream()
+ .map(region -> new DisplayRegion(EEntityKind.valueOf(region.getKind()), region.getLabel(),
+ region.getPersonId() == null ? null : namesById.get(region.getPersonId())))
+ .toList();
+ }
+
+ /** One row for {@code MediaInfoPane}'s persons/animals chips — {@code personName} is {@code null} for an unnamed cluster or any {@link EEntityKind#ANIMAL}. */
+ public record DisplayRegion(EEntityKind kind, @Nullable String label, @Nullable String personName) {
+ }
+
+ /**
+ * Replaces every region {@code mediaFileId} previously had with {@code regions} — delete-then-reinsert,
+ * the same convention {@code LibraryFolderService.persistTags} uses for {@code media_file_tag}, since a
+ * re-detection's region set is typically small and unrelated row-by-row diffing buys nothing.
+ *
+ * Every inserted row starts unlinked ({@code personId = null}) and unconfirmed — see
+ * {@link MediaFaceRegion#confirmed()}'s own javadoc for why automatic detection never confirms a region
+ * itself.
+ */
+ public void replaceRegionsFor(Long mediaFileId, List regions) {
+ repository.deleteByMediaFileIdIn(List.of(mediaFileId));
+ if (regions.isEmpty()) {
+ return;
+ }
+ Instant now = Instant.now();
+ List entities = regions.stream().map(region -> toEntity(mediaFileId, region, now)).toList();
+ repository.saveAll(entities);
+ }
+
+ private static MediaFaceRegionEntity toEntity(Long mediaFileId, DetectedRegion region, Instant detectedAt) {
+ return MediaFaceRegionEntity.builder()
+ .mediaFileId(mediaFileId)
+ .personId(null)
+ .kind(region.kind().name())
+ .areaX(region.box().x())
+ .areaY(region.box().y())
+ .areaW(region.box().w())
+ .areaH(region.box().h())
+ .confidence(region.confidence())
+ .embedding(EmbeddingCodec.toBytes(region.embedding()))
+ .label(region.label())
+ .sourceProvider(region.sourceProvider())
+ .confirmed(false)
+ .detectedAt(detectedAt)
+ .build();
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/IRecognitionService.java b/src/main/java/org/icroco/pholio/infra/recognition/IRecognitionService.java
new file mode 100644
index 0000000..5b53089
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/IRecognitionService.java
@@ -0,0 +1,18 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.recognition.RecognitionResult;
+
+import java.awt.image.BufferedImage;
+
+/**
+ * Finds faces and animals in one already-decoded photo — the local/remote provider seam, mirroring
+ * {@code IPlaceSearchService}. {@link LocalRecognitionService} answers entirely offline, delegating to
+ * whichever {@code engine} package implementation is wired (DJL/ONNX today, swappable later without
+ * touching this seam); {@link RecognitionService} is the {@code @Primary} bean actually injected everywhere,
+ * routing each call to either the local engines or a user-configured remote provider (see its own javadoc)
+ * so the caller never needs to know which one actually answered.
+ */
+public interface IRecognitionService {
+
+ RecognitionResult analyze(BufferedImage image);
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/LocalRecognitionService.java b/src/main/java/org/icroco/pholio/infra/recognition/LocalRecognitionService.java
new file mode 100644
index 0000000..0033d1d
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/LocalRecognitionService.java
@@ -0,0 +1,35 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+import org.icroco.pholio.domain.recognition.RecognitionResult;
+import org.icroco.pholio.infra.recognition.engine.IFaceDetectionEngine;
+import org.icroco.pholio.infra.recognition.engine.IObjectDetectionEngine;
+import org.springframework.stereotype.Component;
+
+import java.awt.image.BufferedImage;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * {@link IRecognitionService} answered entirely by the local engines — no network involved. The class name
+ * itself is the "runs locally" signal {@link RecognitionService}'s own javadoc refers to, the same
+ * convention {@code LocalPlaceSearchService} uses for geocoding.
+ */
+@Component
+public class LocalRecognitionService implements IRecognitionService {
+
+ private final IFaceDetectionEngine faceEngine;
+ private final IObjectDetectionEngine animalEngine;
+
+ public LocalRecognitionService(IFaceDetectionEngine faceEngine, IObjectDetectionEngine animalEngine) {
+ this.faceEngine = faceEngine;
+ this.animalEngine = animalEngine;
+ }
+
+ @Override
+ public RecognitionResult analyze(BufferedImage image) {
+ List regions = new ArrayList<>(faceEngine.detectFaces(image));
+ regions.addAll(animalEngine.detectAnimals(image));
+ return new RecognitionResult(regions);
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/MediaFaceDetectionBackfillTask.java b/src/main/java/org/icroco/pholio/infra/recognition/MediaFaceDetectionBackfillTask.java
new file mode 100644
index 0000000..f817405
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/MediaFaceDetectionBackfillTask.java
@@ -0,0 +1,88 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
+import org.icroco.pholio.infra.library.LibraryFolderService;
+import org.icroco.pholio.infra.persistence.folder.MediaFileEntity;
+import org.icroco.pholio.infra.persistence.folder.MediaFileMapper;
+import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
+import org.icroco.pholio.infra.preferences.AppPreferences;
+import org.icroco.pholio.infra.scheduling.IStartupTask;
+import org.icroco.pholio.infra.task.TaskService;
+import org.icroco.pholio.infra.task.TaskType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.DependsOn;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Backfills face/animal detection, at every application start, for every {@code media_file} row in the
+ * currently open library still missing {@link EMediaFileProcessingFlag#FACE_DETECTED} — the library-wide
+ * catch-up counterpart to {@link MediaFileRecognitionTrigger}'s per-import/per-sync hook, for files that
+ * predate this feature or whose earlier detection attempt never completed.
+ *
+ * Unlike {@code MediaFileProcessingFlagsBackfillTask} (a cheap flag-only fix), this dispatches real
+ * detection work, so it is submitted as a single, visible {@code TaskType.IMAGE_ANALYSIS} batch
+ * (not silent) — the "N files remaining" progress this feature is explicitly meant to show in the status
+ * bar, rather than a silent background fix.
+ *
+ *
{@code @Order(1)}: runs after {@code MediaFileProcessingFlagsBackfillTask} (@Order(0)), so this reads
+ * flags that backfill has already reconciled with reality.
+ *
+ *
{@code @DependsOn("libraryService")} for the same reason as that task: {@code media_file} lives in the
+ * per-library routing datasource, which must already point at an open library.
+ */
+@Component
+@DependsOn("libraryService")
+@Order(1)
+public class MediaFaceDetectionBackfillTask implements IStartupTask {
+
+ private static final Logger log = LoggerFactory.getLogger(MediaFaceDetectionBackfillTask.class);
+
+ private final AppPreferences preferences;
+ private final MediaFileRepository mediaFileRepository;
+ private final MediaFileMapper mediaFileMapper;
+ private final LibraryFolderService libraryFolderService;
+ private final MediaRecognitionService mediaRecognitionService;
+ private final TaskService taskService;
+
+ public MediaFaceDetectionBackfillTask(AppPreferences preferences, MediaFileRepository mediaFileRepository,
+ MediaFileMapper mediaFileMapper, LibraryFolderService libraryFolderService,
+ MediaRecognitionService mediaRecognitionService, TaskService taskService) {
+ this.preferences = preferences;
+ this.mediaFileRepository = mediaFileRepository;
+ this.mediaFileMapper = mediaFileMapper;
+ this.libraryFolderService = libraryFolderService;
+ this.mediaRecognitionService = mediaRecognitionService;
+ this.taskService = taskService;
+ }
+
+ @Override
+ public boolean shouldRun() {
+ return preferences.getValueOr("recognition", "enabled", Boolean.class, true);
+ }
+
+ @Override
+ public void run() {
+ List pending = mediaFileRepository.findAll().stream()
+ .filter(entity -> !mediaFileMapper.toDomain(entity)
+ .hasProcessingFlag(EMediaFileProcessingFlag.FACE_DETECTED))
+ .toList();
+ if (pending.isEmpty()) {
+ log.debug("No media file pending face/animal detection");
+ return;
+ }
+ TaskService.BatchTask batch = taskService.submitBatch(TaskType.IMAGE_ANALYSIS, "Detecting faces & animals", pending.size(), false);
+ pending.forEach(entity -> {
+ Long mediaFileId = Objects.requireNonNull(entity.getId(), "A persisted MediaFileEntity always has an id");
+ libraryFolderService.absolutePathOf(mediaFileMapper.toDomain(entity))
+ .ifPresentOrElse(
+ absolute -> taskService.execute(TaskType.IMAGE_ANALYSIS,
+ () -> mediaRecognitionService.detect(mediaFileId, absolute, batch)),
+ batch::completedOne);
+ });
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/MediaFileRecognitionTrigger.java b/src/main/java/org/icroco/pholio/infra/recognition/MediaFileRecognitionTrigger.java
new file mode 100644
index 0000000..eda7740
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/MediaFileRecognitionTrigger.java
@@ -0,0 +1,65 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
+import org.icroco.pholio.domain.library.MediaFile;
+import org.icroco.pholio.infra.library.LibraryFolderService;
+import org.icroco.pholio.infra.library.MediaFileAnalyzedEvent;
+import org.icroco.pholio.infra.persistence.folder.MediaFileMapper;
+import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
+import org.icroco.pholio.infra.preferences.AppPreferences;
+import org.icroco.pholio.infra.task.TaskService;
+import org.icroco.pholio.infra.task.TaskType;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+
+import java.util.Objects;
+
+/**
+ * Runs face/animal detection for a media file right after {@code MediaAnalysisService} finishes its own
+ * thumbnail+phash pass — covers both a freshly imported file and a re-synced, modified one, since
+ * {@code LibraryFolderService.reimportModifiedFile} already calls {@code generateThumbnail} unconditionally,
+ * whose {@link MediaFileAnalyzedEvent} this listens to either way. No separate hook into
+ * {@code SyncReportService} is needed.
+ *
+ * Skips already-detected files whose pixels are unchanged ({@link MediaFileAnalyzedEvent#pixelsLikelyUnchanged()}
+ * — a metadata-only edit, e.g. a rating change, re-fires this event but must not re-run detection.
+ */
+@Component
+public class MediaFileRecognitionTrigger {
+
+ private final AppPreferences preferences;
+ private final MediaFileRepository mediaFileRepository;
+ private final MediaFileMapper mediaFileMapper;
+ private final LibraryFolderService libraryFolderService;
+ private final MediaRecognitionService mediaRecognitionService;
+ private final TaskService taskService;
+
+ public MediaFileRecognitionTrigger(AppPreferences preferences, MediaFileRepository mediaFileRepository,
+ MediaFileMapper mediaFileMapper, LibraryFolderService libraryFolderService,
+ MediaRecognitionService mediaRecognitionService, TaskService taskService) {
+ this.preferences = preferences;
+ this.mediaFileRepository = mediaFileRepository;
+ this.mediaFileMapper = mediaFileMapper;
+ this.libraryFolderService = libraryFolderService;
+ this.mediaRecognitionService = mediaRecognitionService;
+ this.taskService = taskService;
+ }
+
+ @EventListener
+ public void onAnalyzed(MediaFileAnalyzedEvent event) {
+ if (!preferences.getValueOr("recognition", "enabled", Boolean.class, true)) {
+ return;
+ }
+ mediaFileRepository.findById(event.mediaFileId()).ifPresent(entity -> {
+ MediaFile mediaFile = mediaFileMapper.toDomain(entity);
+ if (mediaFile.hasProcessingFlag(EMediaFileProcessingFlag.FACE_DETECTED) && event.pixelsLikelyUnchanged()) {
+ return;
+ }
+ Long mediaFileId = Objects.requireNonNull(entity.getId(), "A persisted MediaFileEntity always has an id");
+ libraryFolderService.absolutePathOf(mediaFile).ifPresent(absolute -> {
+ TaskService.BatchTask unit = taskService.submitBatch(TaskType.IMAGE_ANALYSIS, "Detecting faces & animals", 1, true);
+ taskService.execute(TaskType.IMAGE_ANALYSIS, () -> mediaRecognitionService.detect(mediaFileId, absolute, unit));
+ });
+ });
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/MediaFileRecognizedEvent.java b/src/main/java/org/icroco/pholio/infra/recognition/MediaFileRecognizedEvent.java
new file mode 100644
index 0000000..6866412
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/MediaFileRecognizedEvent.java
@@ -0,0 +1,9 @@
+package org.icroco.pholio.infra.recognition;
+
+/**
+ * Published once {@code MediaRecognitionService.detect} has replaced a {@code MediaFile}'s detected
+ * regions — {@code MediaInfoPane}'s persons/animals row listens for this to refresh if the file it is
+ * currently showing is the one just analyzed.
+ */
+public record MediaFileRecognizedEvent(Long mediaFileId) {
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/MediaRecognitionService.java b/src/main/java/org/icroco/pholio/infra/recognition/MediaRecognitionService.java
new file mode 100644
index 0000000..a4043bd
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/MediaRecognitionService.java
@@ -0,0 +1,95 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
+import org.icroco.pholio.domain.library.MediaFile;
+import org.icroco.pholio.domain.media.ImageFormat;
+import org.icroco.pholio.domain.recognition.RecognitionResult;
+import org.icroco.pholio.infra.media.MediaFormatRegistry;
+import org.icroco.pholio.infra.media.ThumbnailGenerator;
+import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
+import org.icroco.pholio.infra.task.TaskService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.context.annotation.DependsOn;
+import org.springframework.stereotype.Service;
+
+import java.awt.image.BufferedImage;
+import java.nio.file.Path;
+import java.util.Optional;
+
+/**
+ * The per-file face/animal detection pipeline — decode, analyze, persist, cluster, flag, notify. Run off
+ * {@code TaskType.IMAGE_ANALYSIS} by both {@link MediaFileRecognitionTrigger} (import/re-sync) and
+ * {@link MediaFaceDetectionBackfillTask} (startup backlog).
+ *
+ *
Decodes the original file rather than reusing the small cached thumbnail, for accuracy on small/distant
+ * faces — more expensive than the thumbnail path {@code MediaAnalysisService} optimizes for reuse; see the
+ * implementation plan's own risk notes if this needs revisiting against very large libraries.
+ *
+ *
{@code @DependsOn("libraryService")} for the same reason as {@code MediaAnalysisService}: the routing
+ * datasource must already point at an open library before this touches a repository.
+ */
+@Service
+@DependsOn("libraryService")
+public class MediaRecognitionService {
+
+ private static final Logger log = LoggerFactory.getLogger(MediaRecognitionService.class);
+
+ private final MediaFormatRegistry formats;
+ private final ThumbnailGenerator thumbnailGenerator;
+ private final IRecognitionService recognitionService;
+ private final FaceRegionQueryService faceRegionQueryService;
+ private final FaceClusteringService faceClusteringService;
+ private final MediaFileRepository mediaFileRepository;
+ private final ApplicationEventPublisher publisher;
+
+ public MediaRecognitionService(MediaFormatRegistry formats, ThumbnailGenerator thumbnailGenerator,
+ IRecognitionService recognitionService, FaceRegionQueryService faceRegionQueryService,
+ FaceClusteringService faceClusteringService, MediaFileRepository mediaFileRepository,
+ ApplicationEventPublisher publisher) {
+ this.formats = formats;
+ this.thumbnailGenerator = thumbnailGenerator;
+ this.recognitionService = recognitionService;
+ this.faceRegionQueryService = faceRegionQueryService;
+ this.faceClusteringService = faceClusteringService;
+ this.mediaFileRepository = mediaFileRepository;
+ this.publisher = publisher;
+ }
+
+ /**
+ * @param batchTask completed exactly once, in a {@code finally}, whatever the outcome — the same
+ * discipline {@code MediaAnalysisService.hashAndPublish} follows for its own batch.
+ */
+ public void detect(Long mediaFileId, Path absolute, TaskService.BatchTask batchTask) {
+ try {
+ Optional format = formats.formatOf(absolute);
+ if (format.isEmpty()) {
+ log.warn("'{}' is no longer a recognised format; skipping recognition", absolute);
+ return;
+ }
+ Optional decoded = thumbnailGenerator.decode(absolute, format.get());
+ if (decoded.isEmpty()) {
+ log.debug("No pixels obtainable for '{}'; no recognition run", absolute);
+ return;
+ }
+ BufferedImage oriented = thumbnailGenerator.applyOrientation(decoded.get(), thumbnailGenerator.orientationOf(absolute));
+
+ RecognitionResult result = recognitionService.analyze(oriented);
+ faceRegionQueryService.replaceRegionsFor(mediaFileId, result.regions());
+ faceClusteringService.clusterUnnamedPersons();
+
+ mediaFileRepository.findById(mediaFileId).ifPresent(entity -> {
+ entity.setProcessingFlags(MediaFile.withBit(entity.getProcessingFlags(), EMediaFileProcessingFlag.FACE_DETECTED));
+ mediaFileRepository.save(entity);
+ });
+ publisher.publishEvent(new MediaFileRecognizedEvent(mediaFileId));
+ }
+ catch (RuntimeException e) {
+ log.warn("Face/animal recognition failed unexpectedly for '{}': {}", absolute, e.toString(), e);
+ }
+ finally {
+ batchTask.completedOne();
+ }
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/RecognitionProviderConfig.java b/src/main/java/org/icroco/pholio/infra/recognition/RecognitionProviderConfig.java
new file mode 100644
index 0000000..9d041e3
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/RecognitionProviderConfig.java
@@ -0,0 +1,15 @@
+package org.icroco.pholio.infra.recognition;
+
+/**
+ * One user-configured remote recognition provider, as stored (a JSON list of these) in the
+ * {@code recognition.providers-json} preference — see {@link RecognitionService} for how it is picked and
+ * used, and {@link ERecognitionProviderKind} for the wire contract it must speak.
+ *
+ * @param name shown in the "active provider" picker; also the key {@code recognition.active-provider}
+ * stores to select this config
+ * @param urlTemplate the endpoint to {@code POST} the analysis request to
+ * @param apiKey sent as-is in the request body; the provider validates it however it wants
+ * @param kind which wire contract to speak — see {@link ERecognitionProviderKind}
+ */
+public record RecognitionProviderConfig(String name, String urlTemplate, String apiKey, ERecognitionProviderKind kind) {
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/RecognitionService.java b/src/main/java/org/icroco/pholio/infra/recognition/RecognitionService.java
new file mode 100644
index 0000000..346eef2
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/RecognitionService.java
@@ -0,0 +1,181 @@
+package org.icroco.pholio.infra.recognition;
+
+import org.icroco.pholio.domain.recognition.BoundingBox;
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+import org.icroco.pholio.domain.recognition.EEntityKind;
+import org.icroco.pholio.domain.recognition.RecognitionResult;
+import org.icroco.pholio.infra.preferences.AppPreferences;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+import tools.jackson.core.type.TypeReference;
+import tools.jackson.databind.JsonNode;
+import tools.jackson.databind.ObjectMapper;
+import tools.jackson.databind.node.ObjectNode;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+/**
+ * The {@code @Primary} {@link IRecognitionService} — the one bean actually injected wherever face/animal
+ * recognition is needed. Routes each call to either {@link LocalRecognitionService} or a user-configured
+ * remote provider, transparently to the caller — the exact same shape {@code PlaceSearchService} uses for
+ * geocoding, just a strict proxy rather than a merge: a remote provider's answer is used as-is, never
+ * combined with the local engines' own.
+ *
+ *
+ * - {@code recognition.active-provider} blank/unset (the default) → {@link LocalRecognitionService},
+ * always.
+ *
- otherwise the matching {@link RecognitionProviderConfig} from {@code recognition.providers-json} is
+ * called over HTTP, per {@link ERecognitionProviderKind}'s wire contract.
+ *
+ *
+ * Falls back to {@link LocalRecognitionService} whenever the configured provider cannot answer — an
+ * unknown/deleted provider name, a non-2xx response, a network failure, a malformed reply — rather than
+ * surfacing an error: local detection is always available, so the background pipeline keeps working offline
+ * the same way it always has.
+ */
+@Component
+@Primary
+public class RecognitionService implements IRecognitionService {
+
+ private static final Logger log = LoggerFactory.getLogger(RecognitionService.class);
+
+ private static final String PREFERENCE_GROUP = "recognition";
+ private static final String ACTIVE_PROVIDER_KEY = "active-provider";
+ private static final String PROVIDERS_KEY = "providers-json";
+
+ private final LocalRecognitionService local;
+ private final AppPreferences preferences;
+ private final ObjectMapper json = new ObjectMapper();
+
+ private final HttpClient httpClient = HttpClient.newBuilder()
+ .connectTimeout(Duration.ofSeconds(5))
+ .build();
+
+ public RecognitionService(LocalRecognitionService local, AppPreferences preferences) {
+ this.local = local;
+ this.preferences = preferences;
+ }
+
+ @Override
+ public RecognitionResult analyze(BufferedImage image) {
+ String activeProviderName = preferences.text(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY).orElse("");
+ if (activeProviderName.isBlank()) {
+ log.debug("No active recognition provider configured, analyzing locally");
+ return local.analyze(image);
+ }
+ log.debug("Active recognition provider is '{}', analyzing remotely", activeProviderName);
+ return providers().stream()
+ .filter(provider -> provider.name().equals(activeProviderName))
+ .findFirst()
+ .map(provider -> analyzeRemote(provider, image))
+ .orElseGet(() -> {
+ log.warn("Active recognition provider '{}' is no longer configured, falling back to local analysis",
+ activeProviderName);
+ return local.analyze(image);
+ });
+ }
+
+ /** The configured provider list, in the order they were added — never {@code null}. Exposed for the Maintenance recognition tab. */
+ public List providers() {
+ String raw = preferences.text(PREFERENCE_GROUP, PROVIDERS_KEY).orElse("[]");
+ try {
+ return json.readValue(raw, new TypeReference>() {});
+ }
+ catch (RuntimeException e) {
+ log.warn("Could not read recognition.providers-json ('{}') as a provider list, treating it as empty", raw, e);
+ return List.of();
+ }
+ }
+
+ /** Writes {@code providers} back to {@code recognition.providers-json} — the Maintenance tab's Save action. */
+ public void saveProviders(List providers) {
+ preferences.setValue(PREFERENCE_GROUP, PROVIDERS_KEY, json.writeValueAsString(providers));
+ }
+
+ /** The active provider's name, or blank for "local only". */
+ public String activeProvider() {
+ return preferences.text(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY).orElse("");
+ }
+
+ /** Writes {@code providerName} (blank for "local only") to {@code recognition.active-provider}. */
+ public void saveActiveProvider(String providerName) {
+ preferences.setValue(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY, providerName);
+ }
+
+ private RecognitionResult analyzeRemote(RecognitionProviderConfig provider, BufferedImage image) {
+ log.info("Calling recognition provider '{}' (kind={})", provider.name(), provider.kind());
+ try {
+ String body = requestBody(provider, image);
+ HttpRequest request = HttpRequest.newBuilder(URI.create(provider.urlTemplate()))
+ .header("Content-Type", "application/json")
+ .timeout(Duration.ofSeconds(20))
+ .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8))
+ .build();
+ HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ log.info("Recognition provider '{}' responded HTTP {}", provider.name(), response.statusCode());
+ if (response.statusCode() != 200) {
+ log.warn("Recognition provider '{}' returned HTTP {}, falling back to local analysis",
+ provider.name(), response.statusCode());
+ return local.analyze(image);
+ }
+ RecognitionResult result = parse(response.body(), provider.name());
+ log.info("Recognition provider '{}' returned {} region(s)", provider.name(), result.regions().size());
+ return result;
+ }
+ catch (IOException e) {
+ log.warn("Could not reach recognition provider '{}', falling back to local analysis", provider.name(), e);
+ return local.analyze(image);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return local.analyze(image);
+ }
+ catch (RuntimeException e) {
+ // A remote service changing shape, or simply misbehaving, must not crash the recognition pipeline.
+ log.warn("Could not parse recognition provider '{}' response, falling back to local analysis", provider.name(), e);
+ return local.analyze(image);
+ }
+ }
+
+ private String requestBody(RecognitionProviderConfig provider, BufferedImage image) throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ ImageIO.write(image, "jpg", buffer);
+ ObjectNode node = json.createObjectNode();
+ node.put("apiKey", provider.apiKey());
+ node.put("width", image.getWidth());
+ node.put("height", image.getHeight());
+ node.put("imageBase64", Base64.getEncoder().encodeToString(buffer.toByteArray()));
+ return json.writeValueAsString(node);
+ }
+
+ private RecognitionResult parse(String body, String providerName) {
+ JsonNode root = json.readTree(body);
+ List regions = new ArrayList<>();
+ for (JsonNode node : root) {
+ regions.add(toDetectedRegion(node, providerName));
+ }
+ return new RecognitionResult(regions);
+ }
+
+ private static DetectedRegion toDetectedRegion(JsonNode node, String providerName) {
+ EEntityKind kind = EEntityKind.valueOf(node.path("kind").asString("PERSON"));
+ BoundingBox box = new BoundingBox(node.path("x").asDouble(), node.path("y").asDouble(),
+ node.path("w").asDouble(), node.path("h").asDouble());
+ return new DetectedRegion(kind, box, node.path("confidence").asDouble(1.0),
+ null, node.path("label").asString(null), providerName);
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/IFaceDetectionEngine.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/IFaceDetectionEngine.java
new file mode 100644
index 0000000..ee8cbc1
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/IFaceDetectionEngine.java
@@ -0,0 +1,17 @@
+package org.icroco.pholio.infra.recognition.engine;
+
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+
+import java.awt.image.BufferedImage;
+import java.util.List;
+
+/**
+ * The local face-detection/embedding engine seam — independent of {@code IRecognitionService}'s own
+ * local-vs-remote seam, so swapping this (DJL/ONNX today, JavaCV/OpenCV DNN or anything else later) never
+ * touches {@code RecognitionService}'s routing logic. Each returned {@link DetectedRegion} carries an
+ * embedding for {@code FaceClusteringService} to group across photos.
+ */
+public interface IFaceDetectionEngine {
+
+ List detectFaces(BufferedImage image);
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/IObjectDetectionEngine.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/IObjectDetectionEngine.java
new file mode 100644
index 0000000..a401be1
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/IObjectDetectionEngine.java
@@ -0,0 +1,21 @@
+package org.icroco.pholio.infra.recognition.engine;
+
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+
+import java.awt.image.BufferedImage;
+import java.util.List;
+
+/**
+ * The local animal-detection engine seam — see {@link IFaceDetectionEngine}'s own javadoc for why this is
+ * kept separate from {@code IRecognitionService}'s local-vs-remote seam. Each returned {@link DetectedRegion}
+ * carries a species {@code label} (e.g. {@code "dog"}), not an embedding — individual animal identity is not
+ * attempted in this iteration.
+ *
+ * Named for animals specifically rather than "objects" in general: generic object recognition is a
+ * separate, later phase, and will get its own method here (or a sibling interface) once designed, without
+ * needing to touch any existing caller.
+ */
+public interface IObjectDetectionEngine {
+
+ List detectAnimals(BufferedImage image);
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/NoopAnimalDetectionEngine.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/NoopAnimalDetectionEngine.java
new file mode 100644
index 0000000..edf07ae
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/NoopAnimalDetectionEngine.java
@@ -0,0 +1,15 @@
+package org.icroco.pholio.infra.recognition.engine;
+
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+
+import java.awt.image.BufferedImage;
+import java.util.List;
+
+/** Placeholder {@link IObjectDetectionEngine} — see {@link NoopFaceDetectionEngine}'s own javadoc. */
+public class NoopAnimalDetectionEngine implements IObjectDetectionEngine {
+
+ @Override
+ public List detectAnimals(BufferedImage image) {
+ return List.of();
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/NoopFaceDetectionEngine.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/NoopFaceDetectionEngine.java
new file mode 100644
index 0000000..a0c47fa
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/NoopFaceDetectionEngine.java
@@ -0,0 +1,19 @@
+package org.icroco.pholio.infra.recognition.engine;
+
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+
+import java.awt.image.BufferedImage;
+import java.util.List;
+
+/**
+ * Placeholder {@link IFaceDetectionEngine} — always finds nothing. Superseded by
+ * {@code YuNetSFaceFaceDetectionEngine} as the real {@code @Component}; kept as a plain (non-Spring) class
+ * for tests that need a face engine stand-in without loading actual ONNX models.
+ */
+public class NoopFaceDetectionEngine implements IFaceDetectionEngine {
+
+ @Override
+ public List detectFaces(BufferedImage image) {
+ return List.of();
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/NonMaxSuppression.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/NonMaxSuppression.java
new file mode 100644
index 0000000..bfa9ad3
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/NonMaxSuppression.java
@@ -0,0 +1,41 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+/** Greedy IoU-based NMS, shared by every detector in this package — no per-class grouping (callers that need it group first). */
+final class NonMaxSuppression {
+
+ private NonMaxSuppression() {
+ }
+
+ record Box(double x1, double y1, double x2, double y2, double score, int index) {
+ }
+
+ /** {@code box.index()} of every survivor, highest score first. */
+ static List suppress(List boxes, double iouThreshold) {
+ List byScoreDesc = boxes.stream().sorted(Comparator.comparingDouble(Box::score).reversed()).toList();
+ List kept = new ArrayList<>();
+ List result = new ArrayList<>();
+ for (Box candidate : byScoreDesc) {
+ boolean overlapsKept = kept.stream().anyMatch(k -> iou(candidate, k) > iouThreshold);
+ if (!overlapsKept) {
+ kept.add(candidate);
+ result.add(candidate.index());
+ }
+ }
+ return result;
+ }
+
+ private static double iou(Box a, Box b) {
+ double x1 = Math.max(a.x1(), b.x1());
+ double y1 = Math.max(a.y1(), b.y1());
+ double x2 = Math.min(a.x2(), b.x2());
+ double y2 = Math.min(a.y2(), b.y2());
+ double inter = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
+ double areaA = (a.x2() - a.x1()) * (a.y2() - a.y1());
+ double areaB = (b.x2() - b.x1()) * (b.y2() - b.y1());
+ return inter / (areaA + areaB - inter + 1e-9);
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/OnnxImageTensors.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/OnnxImageTensors.java
new file mode 100644
index 0000000..c4aeedf
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/OnnxImageTensors.java
@@ -0,0 +1,47 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import java.awt.image.BufferedImage;
+
+/** NCHW float tensor extraction from a {@link BufferedImage} — shared by every engine in this package. */
+final class OnnxImageTensors {
+
+ private OnnxImageTensors() {
+ }
+
+ /**
+ * Raw (unnormalized, 0-255) pixel values in planar NCHW layout — every model bundled here was exported
+ * from OpenCV/PyTorch pipelines that feed raw pixel floats, never {@code /255}-scaled ones.
+ *
+ * @param swapToBgr {@code true} to write the R/G/B planes in B,G,R order — every bundled model here was
+ * trained against OpenCV's native BGR channel order except SFace, which explicitly
+ * swaps to RGB before its own forward pass (see {@code YuNetSFaceFaceDetectionEngine}).
+ * {@link BufferedImage#getRGB} always hands back R/G/B regardless of the source file's
+ * own encoding, so this is the one place that channel order is ever chosen.
+ */
+ static float[] toChwFloats(BufferedImage image, boolean swapToBgr) {
+ int width = image.getWidth();
+ int height = image.getHeight();
+ int plane = width * height;
+ float[] data = new float[3 * plane];
+ for (int y = 0; y < height; y++) {
+ for (int x = 0; x < width; x++) {
+ int rgb = image.getRGB(x, y);
+ int r = (rgb >> 16) & 0xFF;
+ int g = (rgb >> 8) & 0xFF;
+ int b = rgb & 0xFF;
+ int idx = y * width + x;
+ if (swapToBgr) {
+ data[idx] = b;
+ data[plane + idx] = g;
+ data[2 * plane + idx] = r;
+ }
+ else {
+ data[idx] = r;
+ data[plane + idx] = g;
+ data[2 * plane + idx] = b;
+ }
+ }
+ }
+ return data;
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/OnnxModelLoader.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/OnnxModelLoader.java
new file mode 100644
index 0000000..6fc3f65
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/OnnxModelLoader.java
@@ -0,0 +1,59 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import ai.onnxruntime.OrtEnvironment;
+import ai.onnxruntime.OrtException;
+import ai.onnxruntime.OrtSession;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+/**
+ * Opens an {@link OrtSession} for a bundled ONNX model — packaged as a classpath resource under
+ * {@code /models/recognition/}, since ONNX Runtime's Java API needs a real file path (or byte array; a file
+ * path is what lets ONNX Runtime memory-map the weights instead of holding a second copy in the JVM heap).
+ * The resource is extracted to the OS temp directory once and reused on every later call/run — its filename
+ * alone is the cache key, since these bundled models never change without a Pholio version bump.
+ */
+final class OnnxModelLoader {
+
+ private static final OrtEnvironment ENVIRONMENT = OrtEnvironment.getEnvironment();
+
+ private OnnxModelLoader() {
+ }
+
+ static OrtSession load(String classpathResource) {
+ try {
+ Path modelFile = extractToCache(classpathResource);
+ return ENVIRONMENT.createSession(modelFile.toString(), new OrtSession.SessionOptions());
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException("Could not extract bundled ONNX model '" + classpathResource + "'", e);
+ }
+ catch (OrtException e) {
+ throw new IllegalStateException("Could not open ONNX session for '" + classpathResource + "'", e);
+ }
+ }
+
+ private static Path extractToCache(String classpathResource) throws IOException {
+ Path cacheDir = Path.of(System.getProperty("java.io.tmpdir"), "pholio-recognition-models");
+ Files.createDirectories(cacheDir);
+ String fileName = classpathResource.substring(classpathResource.lastIndexOf('/') + 1);
+ Path target = cacheDir.resolve(fileName);
+ if (Files.exists(target)) {
+ return target;
+ }
+ try (InputStream in = OnnxModelLoader.class.getResourceAsStream(classpathResource)) {
+ if (in == null) {
+ throw new IOException("Missing bundled model resource: " + classpathResource);
+ }
+ Path staging = Files.createTempFile(cacheDir, "extract-", ".onnx");
+ Files.copy(in, staging, StandardCopyOption.REPLACE_EXISTING);
+ Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING);
+ }
+ return target;
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/SimilarityTransform.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/SimilarityTransform.java
new file mode 100644
index 0000000..b6544d0
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/SimilarityTransform.java
@@ -0,0 +1,48 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+/**
+ * The least-squares 2D similarity transform (uniform scale + rotation + translation, no shear/reflection)
+ * mapping {@code src} points onto {@code dst} points — used to align a detected face's 5 landmarks onto
+ * SFace's fixed reference layout before cropping. Equivalent to Umeyama's algorithm restricted to the
+ * pure-similarity case, but solved directly as a complex-number linear regression rather than via an SVD:
+ * treating each 2D point as a complex number {@code p = x + iy}, the best-fit {@code dst ≈ a*src + t} (a, t
+ * complex) minimizing squared error has the closed form below — no matrix decomposition needed.
+ */
+final class SimilarityTransform {
+
+ private SimilarityTransform() {
+ }
+
+ /** {@code {aRe, aIm, tRe, tIm}} such that {@code dst ≈ a*src + t} with {@code a = aRe + i*aIm}, {@code t = tRe + i*tIm}. */
+ static double[] estimate(double[][] src, double[][] dst) {
+ int n = src.length;
+ double meanSrcX = 0, meanSrcY = 0, meanDstX = 0, meanDstY = 0;
+ for (int i = 0; i < n; i++) {
+ meanSrcX += src[i][0];
+ meanSrcY += src[i][1];
+ meanDstX += dst[i][0];
+ meanDstY += dst[i][1];
+ }
+ meanSrcX /= n;
+ meanSrcY /= n;
+ meanDstX /= n;
+ meanDstY /= n;
+
+ double numRe = 0, numIm = 0, den = 0;
+ for (int i = 0; i < n; i++) {
+ double sx = src[i][0] - meanSrcX;
+ double sy = src[i][1] - meanSrcY;
+ double dx = dst[i][0] - meanDstX;
+ double dy = dst[i][1] - meanDstY;
+ // conj(s) * d = (sx - i*sy)(dx + i*dy) = (sx*dx + sy*dy) + i*(sx*dy - sy*dx)
+ numRe += sx * dx + sy * dy;
+ numIm += sx * dy - sy * dx;
+ den += sx * sx + sy * sy;
+ }
+ double aRe = numRe / den;
+ double aIm = numIm / den;
+ double tRe = meanDstX - (aRe * meanSrcX - aIm * meanSrcY);
+ double tIm = meanDstY - (aIm * meanSrcX + aRe * meanSrcY);
+ return new double[]{ aRe, aIm, tRe, tIm };
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/YoloXAnimalDetectionEngine.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/YoloXAnimalDetectionEngine.java
new file mode 100644
index 0000000..4c7f7a2
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/YoloXAnimalDetectionEngine.java
@@ -0,0 +1,184 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import ai.onnxruntime.OnnxTensor;
+import ai.onnxruntime.OrtEnvironment;
+import ai.onnxruntime.OrtException;
+import ai.onnxruntime.OrtSession;
+import jakarta.annotation.PreDestroy;
+import org.icroco.pholio.domain.recognition.BoundingBox;
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+import org.icroco.pholio.domain.recognition.EEntityKind;
+import org.icroco.pholio.infra.recognition.engine.IObjectDetectionEngine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.awt.Color;
+import java.awt.Graphics2D;
+import java.awt.RenderingHints;
+import java.awt.image.BufferedImage;
+import java.nio.FloatBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * {@link IObjectDetectionEngine} backed by YOLOX-Nano
+ * (Apache-2.0, Megvii), a general 80-class COCO detector filtered down to the animal classes — species label
+ * only, no individual animal identity in this iteration. Preprocessing (letterbox, gray padding) and
+ * postprocessing (grid/stride decode, sigmoid already baked into the exported graph) follow YOLOX's own
+ * {@code demo/ONNXRuntime/onnx_inference.py} and {@code yolox/data/data_augment.py} exactly; input/output
+ * tensor names and shape were confirmed directly against the bundled {@code .onnx} file via
+ * {@code OrtSession.getInputInfo()}/{@code getOutputInfo()}.
+ */
+@Component
+public class YoloXAnimalDetectionEngine implements IObjectDetectionEngine, AutoCloseable {
+
+ private static final Logger log = LoggerFactory.getLogger(YoloXAnimalDetectionEngine.class);
+
+ private static final String SOURCE_PROVIDER = "local-onnx";
+
+ private static final int INPUT_SIZE = 416;
+ private static final int[] STRIDES = { 8, 16, 32 };
+ /** YOLOX-Nano's own {@code demo/ONNXRuntime} defaults. */
+ private static final double SCORE_THRESHOLD = 0.3;
+ private static final double NMS_THRESHOLD = 0.45;
+
+ /** COCO's 80 class names, official order — only indices {@link #ANIMAL_CLASS_MIN}..{@link #ANIMAL_CLASS_MAX} are ever looked at. */
+ private static final String[] COCO_CLASSES = {
+ "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
+ "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
+ "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
+ "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
+ "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+ "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
+ "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
+ "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
+ "hair drier", "toothbrush"
+ };
+ private static final int ANIMAL_CLASS_MIN = 14; // bird
+ private static final int ANIMAL_CLASS_MAX = 23; // giraffe
+
+ private final OrtEnvironment environment = OrtEnvironment.getEnvironment();
+ private final OrtSession session;
+
+ public YoloXAnimalDetectionEngine() {
+ this.session = OnnxModelLoader.load("/models/recognition/yolox_nano.onnx");
+ }
+
+ @Override
+ public List detectAnimals(BufferedImage image) {
+ try {
+ List detections = detectRaw(image);
+ List regions = new ArrayList<>(detections.size());
+ for (RawDetection d : detections) {
+ double boxWidth = d.x2() - d.x1();
+ double boxHeight = d.y2() - d.y1();
+ BoundingBox box = new BoundingBox((d.x1() + boxWidth / 2) / image.getWidth(),
+ (d.y1() + boxHeight / 2) / image.getHeight(),
+ boxWidth / image.getWidth(), boxHeight / image.getHeight());
+ regions.add(new DetectedRegion(EEntityKind.ANIMAL, box, d.score(), null, d.label(), SOURCE_PROVIDER));
+ }
+ return regions;
+ }
+ catch (OrtException e) {
+ log.warn("Animal detection failed unexpectedly: {}", e.toString(), e);
+ return List.of();
+ }
+ }
+
+ private record RawDetection(double x1, double y1, double x2, double y2, double score, String label) {
+ }
+
+ private List detectRaw(BufferedImage image) throws OrtException {
+ Letterbox letterbox = letterbox(image, INPUT_SIZE);
+ float[] chw = OnnxImageTensors.toChwFloats(letterbox.image(), true); // YOLOX expects BGR, raw 0-255 (no /255 normalization)
+
+ List candidates;
+ try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, INPUT_SIZE, INPUT_SIZE });
+ OrtSession.Result result = session.run(Map.of("images", input))) {
+ float[][][] output = (float[][][]) result.get("output").orElseThrow().getValue();
+ candidates = decode(output[0]);
+ }
+
+ List boxes = new ArrayList<>(candidates.size());
+ for (int i = 0; i < candidates.size(); i++) {
+ RawDetection c = candidates.get(i);
+ boxes.add(new NonMaxSuppression.Box(c.x1(), c.y1(), c.x2(), c.y2(), c.score(), i));
+ }
+ List kept = NonMaxSuppression.suppress(boxes, NMS_THRESHOLD);
+
+ List scaled = new ArrayList<>(kept.size());
+ for (int index : kept) {
+ RawDetection c = candidates.get(index);
+ scaled.add(new RawDetection(c.x1() / letterbox.ratio(), c.y1() / letterbox.ratio(),
+ c.x2() / letterbox.ratio(), c.y2() / letterbox.ratio(), c.score(), c.label()));
+ }
+ return scaled;
+ }
+
+ /** {@code predictions}: {@code [numAnchors][85]} — 4 box + 1 objectness + 80 class scores, sigmoid already applied in-graph. */
+ private static List decode(float[][] predictions) {
+ List out = new ArrayList<>();
+ int offset = 0;
+ for (int stride : STRIDES) {
+ int side = INPUT_SIZE / stride;
+ for (int gy = 0; gy < side; gy++) {
+ for (int gx = 0; gx < side; gx++) {
+ float[] pred = predictions[offset + gy * side + gx];
+ double cx = (pred[0] + gx) * stride;
+ double cy = (pred[1] + gy) * stride;
+ double w = Math.exp(pred[2]) * stride;
+ double h = Math.exp(pred[3]) * stride;
+ double obj = pred[4];
+
+ int bestClass = -1;
+ double bestScore = 0;
+ for (int c = ANIMAL_CLASS_MIN; c <= ANIMAL_CLASS_MAX; c++) {
+ double classScore = pred[5 + c];
+ if (classScore > bestScore) {
+ bestScore = classScore;
+ bestClass = c;
+ }
+ }
+ double score = obj * bestScore;
+ if (score < SCORE_THRESHOLD) {
+ continue;
+ }
+ out.add(new RawDetection(cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2, score, COCO_CLASSES[bestClass]));
+ }
+ }
+ offset += side * side;
+ }
+ return out;
+ }
+
+ private record Letterbox(BufferedImage image, double ratio) {
+ }
+
+ /** Resizes preserving aspect ratio onto a {@code target}×{@code target} gray (114,114,114) canvas, content anchored top-left — YOLOX's own {@code preproc}. */
+ private static Letterbox letterbox(BufferedImage source, int target) {
+ double ratio = Math.min(target / (double) source.getHeight(), target / (double) source.getWidth());
+ int newW = Math.round((float) (source.getWidth() * ratio));
+ int newH = Math.round((float) (source.getHeight() * ratio));
+
+ BufferedImage canvas = new BufferedImage(target, target, BufferedImage.TYPE_INT_RGB);
+ Graphics2D g = canvas.createGraphics();
+ try {
+ g.setColor(new Color(114, 114, 114));
+ g.fillRect(0, 0, target, target);
+ g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
+ g.drawImage(source, 0, 0, newW, newH, null);
+ }
+ finally {
+ g.dispose();
+ }
+ return new Letterbox(canvas, ratio);
+ }
+
+ @Override
+ @PreDestroy
+ public void close() throws OrtException {
+ session.close();
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/YuNetSFaceFaceDetectionEngine.java b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/YuNetSFaceFaceDetectionEngine.java
new file mode 100644
index 0000000..d825a40
--- /dev/null
+++ b/src/main/java/org/icroco/pholio/infra/recognition/engine/onnx/YuNetSFaceFaceDetectionEngine.java
@@ -0,0 +1,233 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import ai.onnxruntime.OnnxTensor;
+import ai.onnxruntime.OrtEnvironment;
+import ai.onnxruntime.OrtException;
+import ai.onnxruntime.OrtSession;
+import jakarta.annotation.PreDestroy;
+import org.icroco.pholio.domain.recognition.BoundingBox;
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+import org.icroco.pholio.domain.recognition.EEntityKind;
+import org.icroco.pholio.infra.recognition.engine.IFaceDetectionEngine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.awt.Graphics2D;
+import java.awt.RenderingHints;
+import java.awt.geom.AffineTransform;
+import java.awt.image.BufferedImage;
+import java.nio.FloatBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * {@link IFaceDetectionEngine} backed by two bundled ONNX models, chained: YuNet
+ * (bounding box + 5 landmarks) then SFace
+ * (128-d embedding, run on a landmark-aligned 112×112 crop) — both from OpenCV Zoo, MIT/Apache-2.0 licensed.
+ * Every input tensor shape, output tensor name, decode formula and reference landmark constant below was
+ * either taken verbatim from OpenCV's own {@code face_detect.cpp}/{@code face_recognize.cpp} (the C++ code
+ * these ONNX graphs were designed to be driven by) or confirmed directly against the bundled {@code .onnx}
+ * files via {@code OrtSession.getInputInfo()}/{@code getOutputInfo()} — not guessed.
+ *
+ * {@code detectFaces} always runs both models: a face with no embedding would be useless to
+ * {@code FaceClusteringService}, so there is no "detect only" mode to expose.
+ */
+@Component
+public class YuNetSFaceFaceDetectionEngine implements IFaceDetectionEngine, AutoCloseable {
+
+ private static final Logger log = LoggerFactory.getLogger(YuNetSFaceFaceDetectionEngine.class);
+
+ private static final String SOURCE_PROVIDER = "local-onnx";
+
+ // --- YuNet: fixed 640x640 input (confirmed via OrtSession.getInputInfo()), 3 strides. ---
+ private static final int YUNET_INPUT_SIZE = 640;
+ private static final int[] YUNET_STRIDES = { 8, 16, 32 };
+ /** opencv_zoo's own demo.py defaults — kept identical rather than re-tuned without reference photos to validate against. */
+ private static final double YUNET_SCORE_THRESHOLD = 0.6;
+ private static final double YUNET_NMS_THRESHOLD = 0.3;
+
+ // --- SFace: fixed 112x112 aligned input, 128-d output "fc1". ---
+ private static final int SFACE_CROP_SIZE = 112;
+
+ /**
+ * SFace's own reference layout (right eye, left eye, nose tip, right mouth corner, left mouth corner) in
+ * 112×112 space — quoted verbatim from OpenCV's {@code face_recognize.cpp}. YuNet's 5 landmarks decode in
+ * this exact same order, so no reordering is needed between the two models.
+ */
+ private static final double[][] SFACE_REFERENCE_LANDMARKS = {
+ { 38.2946, 51.6963 }, { 73.5318, 51.5014 }, { 56.0252, 71.7366 }, { 41.5493, 92.3655 }, { 70.7299, 92.2041 }
+ };
+
+ private final OrtEnvironment environment = OrtEnvironment.getEnvironment();
+ private final OrtSession yunetSession;
+ private final OrtSession sfaceSession;
+
+ public YuNetSFaceFaceDetectionEngine() {
+ this.yunetSession = OnnxModelLoader.load("/models/recognition/face_detection_yunet_2023mar.onnx");
+ this.sfaceSession = OnnxModelLoader.load("/models/recognition/face_recognition_sface_2021dec.onnx");
+ }
+
+ @Override
+ public List detectFaces(BufferedImage image) {
+ try {
+ List faces = detectRaw(image);
+ List regions = new ArrayList<>(faces.size());
+ for (RawFace face : faces) {
+ float[] embedding = embed(image, face.landmarks());
+ double boxWidth = face.x2() - face.x1();
+ double boxHeight = face.y2() - face.y1();
+ BoundingBox box = new BoundingBox((face.x1() + boxWidth / 2) / image.getWidth(),
+ (face.y1() + boxHeight / 2) / image.getHeight(),
+ boxWidth / image.getWidth(), boxHeight / image.getHeight());
+ regions.add(new DetectedRegion(EEntityKind.PERSON, box, face.score(), embedding, null, SOURCE_PROVIDER));
+ }
+ return regions;
+ }
+ catch (OrtException e) {
+ log.warn("Face detection/embedding failed unexpectedly: {}", e.toString(), e);
+ return List.of();
+ }
+ }
+
+ private record RawFace(double x1, double y1, double x2, double y2, double score, double[] landmarks) {
+ }
+
+ /** YuNet inference + per-stride decode + NMS, boxes/landmarks already scaled back to {@code image}'s own pixel space. */
+ private List detectRaw(BufferedImage image) throws OrtException {
+ BufferedImage squashed = resize(image, YUNET_INPUT_SIZE, YUNET_INPUT_SIZE);
+ float[] chw = OnnxImageTensors.toChwFloats(squashed, true); // YuNet expects BGR (no swapRB in OpenCV's blobFromImage call)
+
+ List candidates = new ArrayList<>();
+ try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, YUNET_INPUT_SIZE, YUNET_INPUT_SIZE });
+ OrtSession.Result result = yunetSession.run(Map.of("input", input))) {
+ for (int stride : YUNET_STRIDES) {
+ decodeStride(result, stride, candidates);
+ }
+ }
+
+ List boxes = new ArrayList<>(candidates.size());
+ for (int i = 0; i < candidates.size(); i++) {
+ RawFace c = candidates.get(i);
+ boxes.add(new NonMaxSuppression.Box(c.x1(), c.y1(), c.x2(), c.y2(), c.score(), i));
+ }
+ List kept = NonMaxSuppression.suppress(boxes, YUNET_NMS_THRESHOLD);
+
+ double scaleX = image.getWidth() / (double) YUNET_INPUT_SIZE;
+ double scaleY = image.getHeight() / (double) YUNET_INPUT_SIZE;
+ List scaled = new ArrayList<>(kept.size());
+ for (int index : kept) {
+ RawFace c = candidates.get(index);
+ double[] landmarks = new double[10];
+ for (int i = 0; i < 5; i++) {
+ landmarks[2 * i] = c.landmarks()[2 * i] * scaleX;
+ landmarks[2 * i + 1] = c.landmarks()[2 * i + 1] * scaleY;
+ }
+ scaled.add(new RawFace(c.x1() * scaleX, c.y1() * scaleY, c.x2() * scaleX, c.y2() * scaleY, c.score(), landmarks));
+ }
+ return scaled;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void decodeStride(OrtSession.Result result, int stride, List out) throws OrtException {
+ float[][][] cls = (float[][][]) result.get("cls_" + stride).orElseThrow().getValue();
+ float[][][] obj = (float[][][]) result.get("obj_" + stride).orElseThrow().getValue();
+ float[][][] bbox = (float[][][]) result.get("bbox_" + stride).orElseThrow().getValue();
+ float[][][] kps = (float[][][]) result.get("kps_" + stride).orElseThrow().getValue();
+
+ int side = YUNET_INPUT_SIZE / stride;
+ int count = cls[0].length;
+ for (int idx = 0; idx < count; idx++) {
+ int r = idx / side;
+ int c = idx % side;
+ double clsScore = clamp01(cls[0][idx][0]);
+ double objScore = clamp01(obj[0][idx][0]);
+ double score = Math.sqrt(clsScore * objScore);
+ if (score < YUNET_SCORE_THRESHOLD) {
+ continue;
+ }
+ float[] bb = bbox[0][idx];
+ double cx = (c + bb[0]) * stride;
+ double cy = (r + bb[1]) * stride;
+ double w = Math.exp(bb[2]) * stride;
+ double h = Math.exp(bb[3]) * stride;
+
+ float[] kp = kps[0][idx];
+ double[] landmarks = new double[10];
+ for (int n = 0; n < 5; n++) {
+ landmarks[2 * n] = (kp[2 * n] + c) * stride;
+ landmarks[2 * n + 1] = (kp[2 * n + 1] + r) * stride;
+ }
+ out.add(new RawFace(cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2, score, landmarks));
+ }
+ }
+
+ /** Aligns {@code image} onto SFace's reference layout using {@code landmarks} (5 points, original image coordinates), then embeds. */
+ private float[] embed(BufferedImage image, double[] landmarks) throws OrtException {
+ double[][] src = new double[5][2];
+ for (int i = 0; i < 5; i++) {
+ src[i][0] = landmarks[2 * i];
+ src[i][1] = landmarks[2 * i + 1];
+ }
+ double[] t = SimilarityTransform.estimate(src, SFACE_REFERENCE_LANDMARKS);
+
+ BufferedImage aligned = new BufferedImage(SFACE_CROP_SIZE, SFACE_CROP_SIZE, BufferedImage.TYPE_INT_RGB);
+ Graphics2D g = aligned.createGraphics();
+ try {
+ g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
+ // dst = a*src + t, as an AffineTransform: x' = aRe*x - aIm*y + tRe ; y' = aIm*x + aRe*y + tIm
+ g.drawImage(image, new AffineTransform(t[0], t[1], -t[1], t[0], t[2], t[3]), null);
+ }
+ finally {
+ g.dispose();
+ }
+
+ float[] chw = OnnxImageTensors.toChwFloats(aligned, false); // SFace swaps to RGB internally (swapRB=true)
+ try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, SFACE_CROP_SIZE, SFACE_CROP_SIZE });
+ OrtSession.Result result = sfaceSession.run(Map.of("data", input))) {
+ float[][] embedding = (float[][]) result.get("fc1").orElseThrow().getValue();
+ return l2Normalize(embedding[0]);
+ }
+ }
+
+ private static float[] l2Normalize(float[] vector) {
+ double norm = 0;
+ for (float v : vector) {
+ norm += v * v;
+ }
+ norm = Math.sqrt(norm);
+ if (norm == 0) {
+ return vector;
+ }
+ float[] normalized = new float[vector.length];
+ for (int i = 0; i < vector.length; i++) {
+ normalized[i] = (float) (vector[i] / norm);
+ }
+ return normalized;
+ }
+
+ private static double clamp01(double value) {
+ return Math.max(0, Math.min(1, value));
+ }
+
+ private static BufferedImage resize(BufferedImage source, int width, int height) {
+ BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+ Graphics2D g = resized.createGraphics();
+ try {
+ g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
+ g.drawImage(source, 0, 0, width, height, null);
+ }
+ finally {
+ g.dispose();
+ }
+ return resized;
+ }
+
+ @Override
+ @PreDestroy
+ public void close() throws OrtException {
+ yunetSession.close();
+ sfaceSession.close();
+ }
+}
diff --git a/src/main/java/org/icroco/pholio/ui/view/gallery/GalleryView.java b/src/main/java/org/icroco/pholio/ui/view/gallery/GalleryView.java
index 27c76ca..3d56ee1 100644
--- a/src/main/java/org/icroco/pholio/ui/view/gallery/GalleryView.java
+++ b/src/main/java/org/icroco/pholio/ui/view/gallery/GalleryView.java
@@ -22,6 +22,7 @@ import org.icroco.pholio.infra.library.MediaAnalysisService;
import org.icroco.pholio.infra.library.MediaFileService;
import org.icroco.pholio.infra.library.MediaMetadataEditService;
import org.icroco.pholio.infra.preferences.AppPreferences;
+import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.infra.task.TaskType;
import org.icroco.pholio.ui.common.Disposable;
@@ -139,7 +140,8 @@ public class GalleryView extends HBox implements Disposable, SelectionSource {
MediaMetadataEditService metadataEditService,
IPlaceSearchService placeSearchService,
MediaAnalysisService mediaAnalysisService,
- GallerySearchState gallerySearchState) {
+ GallerySearchState gallerySearchState,
+ FaceRegionQueryService faceRegionQueryService) {
this.libraryFolderService = libraryFolderService;
this.preferences = preferences;
this.fullImageCache = fullImageCache;
@@ -155,7 +157,7 @@ public class GalleryView extends HBox implements Disposable, SelectionSource {
galleryPane = new ThumbnailGalleryPane(mediaFileService, taskService, mediaLibraryState, preferences, i18n, imageCache);
underConstructionPane = new UnderConstructionPane();
detailPane = new PhotoDetailPane(fullImageCache);
- mediaInfoPane = new MediaInfoPane(i18n);
+ mediaInfoPane = new MediaInfoPane(i18n, faceRegionQueryService);
galleryPane.setOnOpenRequest((file, sourceThumbnail) -> openDetail(file, sourceThumbnail, null));
galleryPane.setOnRegenerateThumbnails(this::regenerateThumbnails);
galleryPane.setOnSetGpsForFiles(this::openLocationDialogForFiles);
diff --git a/src/main/java/org/icroco/pholio/ui/view/gallery/MediaInfoPane.java b/src/main/java/org/icroco/pholio/ui/view/gallery/MediaInfoPane.java
index 6aa81e2..8fc4b75 100644
--- a/src/main/java/org/icroco/pholio/ui/view/gallery/MediaInfoPane.java
+++ b/src/main/java/org/icroco/pholio/ui/view/gallery/MediaInfoPane.java
@@ -17,6 +17,7 @@ import javafx.scene.control.Label;
import javafx.scene.control.ScrollPane;
import javafx.scene.input.Clipboard;
import javafx.scene.input.ClipboardContent;
+import javafx.scene.layout.FlowPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Priority;
import javafx.scene.layout.StackPane;
@@ -27,6 +28,8 @@ import javafx.util.Duration;
import org.icroco.pholio.domain.library.MediaFile;
import org.icroco.pholio.domain.media.GeoLocation;
import org.icroco.pholio.domain.media.MediaMetadata;
+import org.icroco.pholio.domain.recognition.EEntityKind;
+import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.ui.common.Disposable;
import org.icroco.pholio.ui.control.StarRatingControl;
import org.icroco.pholio.ui.i18n.I18nService;
@@ -83,7 +86,8 @@ public class MediaInfoPane extends StackPane implements Disposable {
/** How long {@link #setOpen} takes to grow/shrink this pane's own width. */
private static final Duration SLIDE_DURATION = Duration.millis(240);
- private final I18nService i18n;
+ private final I18nService i18n;
+ private final FaceRegionQueryService faceRegionQueryService;
private final Label title = new Label();
private final Button closeButton = new Button();
@@ -129,8 +133,9 @@ public class MediaInfoPane extends StackPane implements Disposable {
private Consumer onEditLocation = file -> {};
private BiConsumer onEditRating = (file, rating) -> {};
- public MediaInfoPane(I18nService i18n) {
+ public MediaInfoPane(I18nService i18n, FaceRegionQueryService faceRegionQueryService) {
this.i18n = i18n;
+ this.faceRegionQueryService = faceRegionQueryService;
getStyleClass().add("media-info-pane");
// Starts fully collapsed — GalleryView seeds the real open/closed state right after construction.
@@ -253,6 +258,7 @@ public class MediaInfoPane extends StackPane implements Disposable {
nodes.add(ratingRow(file, metadata));
nodes.add(dateRow(file, metadata));
cameraRow(metadata).ifPresent(nodes::add);
+ personsRow(file).ifPresent(nodes::add);
nodes.add(fileRow(file, metadata));
nodes.add(locationRow(file, metadata));
Optional location = metadata.geoLocation();
@@ -333,6 +339,35 @@ public class MediaInfoPane extends StackPane implements Disposable {
return Optional.of(iconRow(Feather.CAMERA, camera.orElse(null), specs.isEmpty() ? null : String.join(" ", specs)));
}
+ /**
+ * Every person/animal {@code FaceRegionQueryService} has on file for {@code file}, one chip each — a
+ * named person shows their name, an unnamed cluster (or the future confirmation panel not having run
+ * yet) shows {@code "gallery.info.unknownPerson"}, and an animal shows its detected species. Read-only:
+ * naming/confirming a region is the future person-management panel's job, not this pane's.
+ */
+ private Optional personsRow(MediaFile file) {
+ Long mediaFileId = file.id();
+ if (mediaFileId == null) {
+ return Optional.empty();
+ }
+ List regions = faceRegionQueryService.findDisplayRegionsFor(mediaFileId);
+ if (regions.isEmpty()) {
+ return Optional.empty();
+ }
+ FlowPane chips = new FlowPane(6, 6);
+ regions.forEach(region -> chips.getChildren().add(personChip(region)));
+ return Optional.of(chips);
+ }
+
+ private Label personChip(FaceRegionQueryService.DisplayRegion region) {
+ String text = region.kind() == EEntityKind.PERSON
+ ? region.personName() != null ? region.personName() : i18n.get("gallery.info.unknownPerson")
+ : region.label() != null ? region.label() : i18n.get("gallery.info.unknownPerson");
+ Label chip = new Label(text);
+ chip.getStyleClass().add("media-info-person-chip");
+ return chip;
+ }
+
private static String shutterLabel(double seconds) {
return seconds >= 1
? String.format(Locale.ROOT, "%.1fs", seconds)
diff --git a/src/main/resources/css/pholio.css b/src/main/resources/css/pholio.css
index 1178b2e..cc678cf 100644
--- a/src/main/resources/css/pholio.css
+++ b/src/main/resources/css/pholio.css
@@ -371,6 +371,15 @@
-fx-padding: 1 4 1 4;
}
+/* One detected person/animal on MediaInfoPane's persons row — a pill, same rounding convention as
+ .media-info-edit-button, so an unnamed cluster ("Unknown person") reads as a real value, not a link. */
+.media-info-person-chip {
+ -fx-background-color: -color-bg-inset;
+ -fx-background-radius: 999px;
+ -fx-padding: 3 10 3 10;
+ -fx-font-size: 12px;
+}
+
/*
* Transient outcome messages, top-right. The layer itself paints nothing: it is a click-through overlay,
* and each toast is an AtlantaFX Notification carrying its own surface.
diff --git a/src/main/resources/db/migration/V13__create_recognition_tables.sql b/src/main/resources/db/migration/V13__create_recognition_tables.sql
new file mode 100644
index 0000000..a9b4980
--- /dev/null
+++ b/src/main/resources/db/migration/V13__create_recognition_tables.sql
@@ -0,0 +1,41 @@
+-- See V1's header comment: identifiers are quoted so H2 keeps them lower snake_case, matching what
+-- Spring Data JDBC generates.
+
+-- One row per recognized identity. Only PERSON rows are linked to from media_face_region in this
+-- iteration — an ANIMAL region never gets one (species lives directly on media_face_region.label, no
+-- per-animal identity yet).
+CREATE TABLE "person"
+(
+ "id" BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ "kind" VARCHAR(16) NOT NULL,
+ "name" VARCHAR(255),
+ "created_at" TIMESTAMP NOT NULL,
+ CONSTRAINT "ck_person_kind" CHECK ("kind" IN ('PERSON', 'ANIMAL'))
+);
+
+-- One row per detected face/animal bounding box. person_id stays NULL until FaceClusteringService (PERSON)
+-- links it; ANIMAL rows never get one. Re-detection replaces a file's rows wholesale (delete-then-reinsert),
+-- the same convention media_file_tag already uses for its own re-scans.
+CREATE TABLE "media_face_region"
+(
+ "id" BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+ "media_file_id" BIGINT NOT NULL REFERENCES "media_file" ("id") ON DELETE CASCADE,
+ "person_id" BIGINT REFERENCES "person" ("id") ON DELETE SET NULL,
+ "kind" VARCHAR(16) NOT NULL,
+ -- Normalized MWG-RS "stArea": x/y is the region's CENTER, w/h its size, fractions of the full image
+ -- (0..1) — identical numbers to what an mwg-rs:Area XMP struct stores, zero conversion on round-trip.
+ "area_x" DOUBLE NOT NULL,
+ "area_y" DOUBLE NOT NULL,
+ "area_w" DOUBLE NOT NULL,
+ "area_h" DOUBLE NOT NULL,
+ "confidence" DOUBLE NOT NULL,
+ "embedding" VARBINARY(8192),
+ "label" VARCHAR(255),
+ "source_provider" VARCHAR(255) NOT NULL,
+ "confirmed" BOOLEAN NOT NULL DEFAULT FALSE,
+ "detected_at" TIMESTAMP NOT NULL,
+ CONSTRAINT "ck_media_face_region_kind" CHECK ("kind" IN ('PERSON', 'ANIMAL'))
+);
+
+CREATE INDEX "ix_media_face_region_media_file" ON "media_face_region" ("media_file_id");
+CREATE INDEX "ix_media_face_region_person" ON "media_face_region" ("person_id");
diff --git a/src/main/resources/messages.properties b/src/main/resources/messages.properties
index fd143a3..0cc6d09 100644
--- a/src/main/resources/messages.properties
+++ b/src/main/resources/messages.properties
@@ -102,6 +102,7 @@ gallery.unknownDate=Unknown date
gallery.info.hash=Hash
gallery.info.noMetadata=No metadata available
gallery.info.addPlace=Add a location
+gallery.info.unknownPerson=Unknown person
gallery.location.edit.title=Add a location
gallery.location.edit.searchPrompt=Search for a place…
gallery.location.edit.hint=Changes to the place a photo was taken are saved to the library and, when the format supports it, to the file itself.
@@ -170,6 +171,10 @@ settings.geocoding.cities5000LastImport=Cities5000 reference data last imported
settings.geocoding.cities5000RowCount=Cities5000 reference row count
settings.geocoding.providersJson=Address search providers
settings.geocoding.activeProvider=Active address search provider
+settings.recognition.enabled=Face/animal recognition enabled
+settings.recognition.providersJson=Recognition providers
+settings.recognition.activeProvider=Active recognition provider
+settings.recognition.clusterThreshold=Person clustering similarity threshold
settings.ai.provider=Provider
settings.ai.endpoint=Endpoint
settings.imports.largeFolderThreshold=Confirm above (files)
diff --git a/src/main/resources/messages_fr.properties b/src/main/resources/messages_fr.properties
index c5aaf9d..4d5b171 100644
--- a/src/main/resources/messages_fr.properties
+++ b/src/main/resources/messages_fr.properties
@@ -104,6 +104,7 @@ gallery.unknownDate=Date inconnue
gallery.info.hash=Hash
gallery.info.noMetadata=Aucune métadonnée disponible
gallery.info.addPlace=Ajouter un lieu
+gallery.info.unknownPerson=Personne inconnue
gallery.location.edit.title=Ajouter un lieu
gallery.location.edit.searchPrompt=Rechercher un lieu…
gallery.location.edit.hint=Les modifications apportées au lieu de prise de vue seront enregistrées dans la photothèque et, si le format le permet, dans le fichier lui-même.
@@ -172,6 +173,10 @@ settings.geocoding.cities5000LastImport=Dernière importation des données de r
settings.geocoding.cities5000RowCount=Nombre de lignes de référence Cities5000
settings.geocoding.providersJson=Fournisseurs de recherche d'adresse
settings.geocoding.activeProvider=Fournisseur de recherche d'adresse actif
+settings.recognition.enabled=Reconnaissance de visages/animaux activée
+settings.recognition.providersJson=Fournisseurs de reconnaissance
+settings.recognition.activeProvider=Fournisseur de reconnaissance actif
+settings.recognition.clusterThreshold=Seuil de similarité pour le regroupement de personnes
settings.ai.provider=Fournisseur
settings.ai.endpoint=Point d'accès
settings.imports.largeFolderThreshold=Confirmer au-delà de (fichiers)
diff --git a/src/main/resources/models/recognition/NOTICE.md b/src/main/resources/models/recognition/NOTICE.md
new file mode 100644
index 0000000..a8cf525
--- /dev/null
+++ b/src/main/resources/models/recognition/NOTICE.md
@@ -0,0 +1,22 @@
+# Bundled recognition models
+
+Third-party ONNX weights bundled with Pholio for local face/animal recognition. None were modified —
+each is used exactly as published upstream.
+
+## face_detection_yunet_2023mar.onnx
+
+- Source: [opencv/opencv_zoo](https://github.com/opencv/opencv_zoo/tree/main/models/face_detection_yunet)
+- License: MIT
+- Used by: `YuNetSFaceFaceDetectionEngine` (face bounding box + 5-point landmarks)
+
+## face_recognition_sface_2021dec.onnx
+
+- Source: [opencv/opencv_zoo](https://github.com/opencv/opencv_zoo/tree/main/models/face_recognition_sface)
+- License: Apache-2.0
+- Used by: `YuNetSFaceFaceDetectionEngine` (128-d face embedding, for `FaceClusteringService`)
+
+## yolox_nano.onnx
+
+- Source: [Megvii-BaseDetection/YOLOX](https://github.com/Megvii-BaseDetection/YOLOX), release `0.1.1rc0`
+- License: Apache-2.0
+- Used by: `YoloXAnimalDetectionEngine` (COCO 80-class detector, filtered to animal classes 14-23)
diff --git a/src/main/resources/models/recognition/face_detection_yunet_2023mar.onnx b/src/main/resources/models/recognition/face_detection_yunet_2023mar.onnx
new file mode 100644
index 0000000..2d8804a
--- /dev/null
+++ b/src/main/resources/models/recognition/face_detection_yunet_2023mar.onnx
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f2383e4dd3cfbb4553ea8718107fc0423210dc964f9f4280604804ed2552fa4
+size 232589
diff --git a/src/main/resources/models/recognition/face_recognition_sface_2021dec.onnx b/src/main/resources/models/recognition/face_recognition_sface_2021dec.onnx
new file mode 100644
index 0000000..5817e55
--- /dev/null
+++ b/src/main/resources/models/recognition/face_recognition_sface_2021dec.onnx
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ba9fbfa01b5270c96627c4ef784da859931e02f04419c829e83484087c34e79
+size 38696353
diff --git a/src/main/resources/models/recognition/yolox_nano.onnx b/src/main/resources/models/recognition/yolox_nano.onnx
new file mode 100644
index 0000000..75579ee
--- /dev/null
+++ b/src/main/resources/models/recognition/yolox_nano.onnx
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c789161ed43c8269fcd4e67c67eeeb4e80c622da2eb296a20bc6007bd18a0b7d
+size 3659407
diff --git a/src/main/resources/preferences.yaml b/src/main/resources/preferences.yaml
index 2cfdb62..618a3aa 100644
--- a/src/main/resources/preferences.yaml
+++ b/src/main/resources/preferences.yaml
@@ -281,6 +281,42 @@ geocoding:
visible: false
editable: false
+# Local/remote face+animal recognition — same shape as `geocoding` above, edited only through Maintenance's
+# recognition tab, never through this generic settings form.
+recognition:
+ enabled:
+ type: BOOLEAN
+ label: settings.recognition.enabled
+ default-value: true
+ visible: false
+ editable: false
+
+ # Edited only through Maintenance's recognition tab — a JSON-encoded list of RecognitionProviderConfig.
+ providers-json:
+ type: STRING
+ label: settings.recognition.providersJson
+ default-value: "[]"
+ visible: false
+ editable: false
+
+ # The active RecognitionProviderConfig's name, or blank to mean "local only" — see RecognitionService.
+ active-provider:
+ type: STRING
+ label: settings.recognition.activeProvider
+ default-value: ""
+ visible: false
+ editable: false
+
+ # Cosine-similarity threshold above which FaceClusteringService links an unnamed face to an existing
+ # Person cluster rather than minting a new one. 0.363 is SFace's own calibrated same-identity threshold
+ # (OpenCV Zoo's face_recognition_sface README) — the embedding is SFace's, so its own threshold applies.
+ person-cluster-threshold:
+ type: DOUBLE
+ label: settings.recognition.clusterThreshold
+ default-value: 0.363
+ visible: false
+ editable: false
+
# Onboarding, shown once on the first run. Not offered in the settings view: there is nothing to configure,
# only a fact to remember once the coach-mark sequence has been dismissed or completed.
onboarding:
diff --git a/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/NonMaxSuppressionTest.java b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/NonMaxSuppressionTest.java
new file mode 100644
index 0000000..03b56ec
--- /dev/null
+++ b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/NonMaxSuppressionTest.java
@@ -0,0 +1,43 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class NonMaxSuppressionTest {
+
+ @Test
+ void suppressesTheLowerScoredOfTwoHeavilyOverlappingBoxes() {
+ List boxes = List.of(
+ new NonMaxSuppression.Box(0, 0, 10, 10, 0.9, 0),
+ new NonMaxSuppression.Box(1, 1, 11, 11, 0.5, 1));
+
+ List kept = NonMaxSuppression.suppress(boxes, 0.3);
+
+ assertThat(kept).containsExactly(0);
+ }
+
+ @Test
+ void keepsBothOfTwoNonOverlappingBoxes() {
+ List boxes = List.of(
+ new NonMaxSuppression.Box(0, 0, 10, 10, 0.9, 0),
+ new NonMaxSuppression.Box(100, 100, 110, 110, 0.5, 1));
+
+ List kept = NonMaxSuppression.suppress(boxes, 0.3);
+
+ assertThat(kept).containsExactlyInAnyOrder(0, 1);
+ }
+
+ @Test
+ void ordersSurvivorsHighestScoreFirst() {
+ List boxes = List.of(
+ new NonMaxSuppression.Box(0, 0, 10, 10, 0.4, 0),
+ new NonMaxSuppression.Box(100, 100, 110, 110, 0.9, 1));
+
+ List kept = NonMaxSuppression.suppress(boxes, 0.3);
+
+ assertThat(kept).containsExactly(1, 0);
+ }
+}
diff --git a/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/SimilarityTransformTest.java b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/SimilarityTransformTest.java
new file mode 100644
index 0000000..7110b1a
--- /dev/null
+++ b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/SimilarityTransformTest.java
@@ -0,0 +1,58 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import org.assertj.core.api.SoftAssertions;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.within;
+
+class SimilarityTransformTest {
+
+ @Test
+ void recoversAnExactUniformScaleAndTranslation() {
+ double[][] src = { { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } };
+ double[][] dst = new double[src.length][2];
+ for (int i = 0; i < src.length; i++) {
+ dst[i][0] = 2 * src[i][0] + 10;
+ dst[i][1] = 2 * src[i][1] + 5;
+ }
+
+ double[] t = SimilarityTransform.estimate(src, dst);
+
+ SoftAssertions.assertSoftly(softly -> {
+ softly.assertThat(t[0]).as("scale (real part)").isCloseTo(2.0, within(1e-9));
+ softly.assertThat(t[1]).as("rotation (imaginary part)").isCloseTo(0.0, within(1e-9));
+ softly.assertThat(t[2]).as("translation x").isCloseTo(10.0, within(1e-9));
+ softly.assertThat(t[3]).as("translation y").isCloseTo(5.0, within(1e-9));
+ });
+ }
+
+ @Test
+ void recoversAnExactRotation() {
+ // 90 degrees counter-clockwise about the origin: (x, y) -> (-y, x)
+ double[][] src = { { 1, 0 }, { 0, 1 }, { -1, 0 }, { 0, -1 } };
+ double[][] dst = { { 0, 1 }, { -1, 0 }, { 0, -1 }, { 1, 0 } };
+
+ double[] t = SimilarityTransform.estimate(src, dst);
+
+ SoftAssertions.assertSoftly(softly -> {
+ softly.assertThat(t[0]).as("scale (real part)").isCloseTo(0.0, within(1e-9));
+ softly.assertThat(t[1]).as("rotation (imaginary part)").isCloseTo(1.0, within(1e-9));
+ softly.assertThat(t[2]).as("translation x").isCloseTo(0.0, within(1e-9));
+ softly.assertThat(t[3]).as("translation y").isCloseTo(0.0, within(1e-9));
+ });
+ }
+
+ @Test
+ void identityWhenSourceAlreadyMatchesDestination() {
+ double[][] points = { { 3, 4 }, { 5, 1 }, { -2, 7 } };
+
+ double[] t = SimilarityTransform.estimate(points, points);
+
+ SoftAssertions.assertSoftly(softly -> {
+ softly.assertThat(t[0]).isCloseTo(1.0, within(1e-9));
+ softly.assertThat(t[1]).isCloseTo(0.0, within(1e-9));
+ softly.assertThat(t[2]).isCloseTo(0.0, within(1e-9));
+ softly.assertThat(t[3]).isCloseTo(0.0, within(1e-9));
+ });
+ }
+}
diff --git a/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/YoloXAnimalDetectionEngineTest.java b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/YoloXAnimalDetectionEngineTest.java
new file mode 100644
index 0000000..7948386
--- /dev/null
+++ b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/YoloXAnimalDetectionEngineTest.java
@@ -0,0 +1,37 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import ai.onnxruntime.OrtException;
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+import org.junit.jupiter.api.Test;
+
+import java.awt.image.BufferedImage;
+import java.util.List;
+import java.util.Random;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Smoke test — see {@link YuNetSFaceFaceDetectionEngineTest}'s own javadoc for why this isn't an accuracy test. */
+class YoloXAnimalDetectionEngineTest {
+
+ @Test
+ void loadsBundledModelAndRunsInferenceWithoutThrowing() throws OrtException {
+ try (YoloXAnimalDetectionEngine engine = new YoloXAnimalDetectionEngine()) {
+ BufferedImage image = randomImage(640, 480);
+
+ List regions = engine.detectAnimals(image);
+
+ assertThat(regions).isNotNull();
+ }
+ }
+
+ private static BufferedImage randomImage(int width, int height) {
+ BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+ Random random = new Random(42);
+ for (int y = 0; y < height; y++) {
+ for (int x = 0; x < width; x++) {
+ image.setRGB(x, y, random.nextInt(0xFFFFFF));
+ }
+ }
+ return image;
+ }
+}
diff --git a/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/YuNetSFaceFaceDetectionEngineTest.java b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/YuNetSFaceFaceDetectionEngineTest.java
new file mode 100644
index 0000000..d84ef97
--- /dev/null
+++ b/src/test/java/org/icroco/pholio/infra/recognition/engine/onnx/YuNetSFaceFaceDetectionEngineTest.java
@@ -0,0 +1,43 @@
+package org.icroco.pholio.infra.recognition.engine.onnx;
+
+import ai.onnxruntime.OrtException;
+import org.icroco.pholio.domain.recognition.DetectedRegion;
+import org.junit.jupiter.api.Test;
+
+import java.awt.image.BufferedImage;
+import java.util.List;
+import java.util.Random;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * A smoke test, not an accuracy test: no real face photo ships with this repo (nothing to license/attribute,
+ * nothing that could be mistaken for real personal data), so this only pins down that the bundled ONNX
+ * models load and run inference without throwing on this exact JDK/ONNX Runtime combination — the actual
+ * risk flagged in the recognition feature's implementation plan. A synthetic noise image legitimately finds
+ * zero faces; that is the correct, expected answer, not a test gap.
+ */
+class YuNetSFaceFaceDetectionEngineTest {
+
+ @Test
+ void loadsBundledModelsAndRunsInferenceWithoutThrowing() throws OrtException {
+ try (YuNetSFaceFaceDetectionEngine engine = new YuNetSFaceFaceDetectionEngine()) {
+ BufferedImage image = randomImage(640, 480);
+
+ List regions = engine.detectFaces(image);
+
+ assertThat(regions).isNotNull();
+ }
+ }
+
+ private static BufferedImage randomImage(int width, int height) {
+ BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+ Random random = new Random(42);
+ for (int y = 0; y < height; y++) {
+ for (int x = 0; x < width; x++) {
+ image.setRGB(x, y, random.nextInt(0xFFFFFF));
+ }
+ }
+ return image;
+ }
+}
diff --git a/src/test/java/org/icroco/pholio/ui/library/MediaLibraryStateTest.java b/src/test/java/org/icroco/pholio/ui/library/MediaLibraryStateTest.java
index 108118c..650dee3 100644
--- a/src/test/java/org/icroco/pholio/ui/library/MediaLibraryStateTest.java
+++ b/src/test/java/org/icroco/pholio/ui/library/MediaLibraryStateTest.java
@@ -62,7 +62,7 @@ class MediaLibraryStateTest {
boolean readyBefore = state.thumbnailReadyProperty(1L).get();
boolean otherFileReady = state.thumbnailReadyProperty(2L).get();
- runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
+ runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(readyBefore).isFalse();
@@ -97,7 +97,7 @@ class MediaLibraryStateTest {
void folderRemovedClearsEveryThumbnailReadyProperty() throws InterruptedException {
FxTestToolkit.requireToolkit();
MediaLibraryState state = new MediaLibraryState();
- runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
+ runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(state.thumbnailReadyProperty(1L).get()).isTrue();
@@ -136,7 +136,7 @@ class MediaLibraryStateTest {
void libraryChangedClearsEveryThumbnailReadyProperty() throws InterruptedException {
FxTestToolkit.requireToolkit();
MediaLibraryState state = new MediaLibraryState();
- runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
+ runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(state.thumbnailReadyProperty(1L).get()).isTrue();
diff --git a/src/test/java/org/icroco/pholio/ui/shell/NavigationRailTest.java b/src/test/java/org/icroco/pholio/ui/shell/NavigationRailTest.java
index d4ff067..208078b 100644
--- a/src/test/java/org/icroco/pholio/ui/shell/NavigationRailTest.java
+++ b/src/test/java/org/icroco/pholio/ui/shell/NavigationRailTest.java
@@ -13,6 +13,7 @@ import org.icroco.pholio.infra.library.MediaFileService;
import org.icroco.pholio.infra.library.MediaMetadataEditService;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.icroco.pholio.infra.preferences.PreferencesFixture;
+import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.ui.FxTestToolkit;
import org.icroco.pholio.ui.ViewSwitcher;
@@ -345,7 +346,8 @@ class NavigationRailTest {
mock(MediaMetadataEditService.class),
mock(IPlaceSearchService.class),
mock(MediaAnalysisService.class),
- new GallerySearchState());
+ new GallerySearchState(),
+ mock(FaceRegionQueryService.class));
when(context.getBean(GalleryView.class)).thenReturn(galleryView);
when(context.getBean(ModulePlaceholderView.class)).thenReturn(new ModulePlaceholderView(i18n));
return new ViewSwitcher(context, new ViewportSelection());
diff --git a/src/test/java/org/icroco/pholio/ui/view/gallery/GalleryViewTest.java b/src/test/java/org/icroco/pholio/ui/view/gallery/GalleryViewTest.java
index ee30eb3..0c7333a 100644
--- a/src/test/java/org/icroco/pholio/ui/view/gallery/GalleryViewTest.java
+++ b/src/test/java/org/icroco/pholio/ui/view/gallery/GalleryViewTest.java
@@ -22,6 +22,7 @@ import org.icroco.pholio.infra.library.MediaFileService;
import org.icroco.pholio.infra.library.MediaMetadataEditService;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.icroco.pholio.infra.preferences.PreferencesFixture;
+import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.ui.FxTestToolkit;
import org.icroco.pholio.ui.i18n.I18nService;
@@ -87,7 +88,8 @@ class GalleryViewTest {
mock(MediaMetadataEditService.class),
mock(IPlaceSearchService.class),
mock(MediaAnalysisService.class),
- new GallerySearchState()));
+ new GallerySearchState(),
+ mock(FaceRegionQueryService.class)));
}
@Test
@@ -214,7 +216,7 @@ class GalleryViewTest {
new ThumbnailImageCache(localPreferences, taskService),
new FullImageCache(localPreferences, taskService), mock(LibraryFolderService.class),
mock(ModalService.class), mock(MediaMetadataEditService.class), mock(IPlaceSearchService.class),
- mock(MediaAnalysisService.class), new GallerySearchState()));
+ mock(MediaAnalysisService.class), new GallerySearchState(), mock(FaceRegionQueryService.class)));
Stage farStage = onFxThread(() -> {
farView.resize(400, 500);
diff --git a/src/test/java/org/icroco/pholio/ui/view/gallery/ThumbnailGalleryPaneTest.java b/src/test/java/org/icroco/pholio/ui/view/gallery/ThumbnailGalleryPaneTest.java
index dbd0e3e..a2c3b7d 100644
--- a/src/test/java/org/icroco/pholio/ui/view/gallery/ThumbnailGalleryPaneTest.java
+++ b/src/test/java/org/icroco/pholio/ui/view/gallery/ThumbnailGalleryPaneTest.java
@@ -518,7 +518,7 @@ class ThumbnailGalleryPaneTest {
clearInvocations(mediaFileService);
List rowsBefore = pane.rows().getItems();
- runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
+ runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(pane.rows().getItems()).as("same instance: no relayout, only the one cell's slot changed")
@@ -565,7 +565,7 @@ class ThumbnailGalleryPaneTest {
Files.createDirectories(thumbnail.getParent());
Files.write(thumbnail, new byte[]{1, 2, 3});
- runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
+ runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
Node cardAfter = onFxThread(() -> {