feat(recognition): add local face/animal detection pipeline

Add a face/animal recognition feature: local ONNX detection (YuNet+SFace
for faces with automatic person clustering, YOLOX-Nano for animal species)
behind an IRecognitionService seam mirroring IPlaceSearchService's local/
remote proxy pattern, with a separate engine seam so the ONNX backend can
be swapped later. Wired into the import/sync pipeline via TaskType.IMAGE_ANALYSIS
with a visible progress task, a startup backfill for existing libraries, and
a cheap perceptual-hash skip so metadata-only edits don't retrigger detection.
Detected regions are stored (new person/media_face_region tables) and shown
read-only in MediaInfoPane. Bundled ONNX weights (MIT/Apache-2.0) are tracked
via Git LFS.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JzvA5ySQUsYrMUTj7sHxFA
This commit is contained in:
2026-09-13 19:41:04 -04:00
co-authored by Claude Sonnet 5
parent a081dc1165
commit d15c697e7c
59 changed files with 2226 additions and 25 deletions
+4
View File
@@ -19,3 +19,7 @@
*.jpeg binary
*.ico binary
*.icns binary
# Bundled ONNX recognition models (src/main/resources/models/recognition/) — tens of MB each, stored via
# Git LFS rather than directly in history so a plain clone stays small unless LFS objects are pulled.
*.onnx filter=lfs diff=lfs merge=lfs -text
+9 -5
View File
@@ -17,7 +17,7 @@
## Shell
* [ ] nouveau menu pour help / credit
* [ ] prevoir des scenario pour les aides guide
* [ ] prevoir des scenarios pour les aides guide
## Status bar
@@ -33,11 +33,15 @@
## Photos detail
* [ ] implementer le panel info en slide * [ ] Info: ajouter les infos en lecteur * [ ] Info: editer date * [ ] Info: editer lieux
* [ ] afficher star + modifier star
* [ ] zoom in / out sur la photo : fonctionne avec la souris, mais pas terrible avec le trackpad
* [x] tenir compte de l'orientation
* [ ] Animation de L'image qd le panel info arrive
* [x] implementer le panel info en slide
* [x] Info: ajouter les infos en lecteur
* [x] Info: editer date
* [x] Info: editer lieux
* [x] la carte scintille a chaque changement de photos.
* [x] afficher star + modifier star
* [x] tenir compte de l'orientation
* [x] cache image dans PhotoDetailsPane
## Notifications
@@ -60,6 +64,6 @@
* [x] ecrire le gps
* [ ] ecrire les tags
* [x] ecrire la date
* [ ] ecrire rating
* [x] ecrire rating
* [ ] definir un writer pour tous les fichier pas supporte avec exif tool: binaire inclus dans l'app et installation au premier demarrage. notif en erreur si le
binaire n'est plus trouve.
+17
View File
@@ -35,6 +35,16 @@
<byte-buddy.version>1.18.12</byte-buddy.version>
<commons-codec.version>1.22.1</commons-codec.version>
<commons-imaging.version>1.0.0-alpha6</commons-imaging.version>
<!--
Local face/animal recognition (YuNet detection, SFace embedding, YOLOX-Nano animal
detection) runs on Microsoft's own ONNX Runtime Java API directly, not through Deep
Java Library: every model here needs fully custom pre/post-processing (letterboxing,
anchor decoding, NMS, 5-point face alignment), so DJL's Translator/ZooModel machinery
would add an abstraction layer with nothing left for it to actually do. This jar bundles
native libraries for every major desktop platform in one artifact — no per-OS classifier,
no runtime download.
-->
<onnxruntime.version>1.21.1</onnxruntime.version>
<!--
Scene-graph inspector, wired in by DevTools and dormant unless pholio.dev-tools.enabled is
set. Built for Java 21 / JavaFX 23; Pholio runs it on 26 in classpath mode, so it relies on
@@ -256,6 +266,13 @@
<artifactId>commons-codec</artifactId>
<version>${commons-codec.version}</version>
</dependency>
<!-- ==================== Local ML (recognition) ==================== -->
<dependency>
<groupId>com.microsoft.onnxruntime</groupId>
<artifactId>onnxruntime</artifactId>
<version>${onnxruntime.version}</version>
</dependency>
<!-- =========================== Caching =========================== -->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
@@ -8,7 +8,8 @@ package org.icroco.pholio.domain.library;
public enum EMediaFileProcessingFlag {
METADATA_GENERATED(0),
THUMBNAIL_GENERATED(1),
GEO_REFERENCE_GENERATED(2);
GEO_REFERENCE_GENERATED(2),
FACE_DETECTED(3);
private final int bit;
@@ -0,0 +1,10 @@
package org.icroco.pholio.domain.recognition;
/**
* A detected region's location within its image, in the MWG-RS {@code stArea} convention: {@code x}/{@code y}
* is the box's <b>center</b> point, {@code w}/{@code h} its size — all four fractions of the full image's own
* width/height (0..1), not pixels. Chosen deliberately so {@code media_face_region}'s stored columns are
* already in the exact shape an {@code mwg-rs:Area} XMP struct needs on write, with no conversion either way.
*/
public record BoundingBox(double x, double y, double w, double h) {
}
@@ -0,0 +1,24 @@
package org.icroco.pholio.domain.recognition;
import org.jilt.Builder;
import org.jspecify.annotations.Nullable;
/**
* One face or animal found in a photo by an {@code IRecognitionService} call — not yet persisted, and not yet
* linked to any {@link Person}; see {@link MediaFaceRegion} for the persisted, DB-identified counterpart.
*
* @param embedding {@link EEntityKind#PERSON} only — the face embedding vector clustering compares
* across photos to group the same identity; {@code null} for {@link EEntityKind#ANIMAL}
* @param label {@link EEntityKind#ANIMAL} only — the detected species (e.g. {@code "dog"}); {@code
* null} for {@link EEntityKind#PERSON}, whose identity is a {@link Person}, not a label
* @param sourceProvider which {@code IRecognitionService}/engine produced this — {@code "local-djl"}, or a
* configured remote provider's own name
*/
@Builder(factoryMethod = "detectedRegion")
public record DetectedRegion(EEntityKind kind,
BoundingBox box,
double confidence,
float @Nullable [] embedding,
@Nullable String label,
String sourceProvider) {
}
@@ -0,0 +1,7 @@
package org.icroco.pholio.domain.recognition;
/** What a {@link DetectedRegion}/{@link MediaFaceRegion} identifies — a human face or an animal. */
public enum EEntityKind {
PERSON,
ANIMAL
}
@@ -0,0 +1,32 @@
package org.icroco.pholio.domain.recognition;
import org.jilt.Builder;
import org.jspecify.annotations.Nullable;
import java.time.Instant;
/**
* A {@link DetectedRegion} once persisted against a {@code MediaFile} — see {@link DetectedRegion} for what
* each field means; the two differ only in that this one carries a database identity and an optional link to
* the {@link Person} {@code FaceClusteringService} has grouped it under.
*
* @param personId {@code null} until {@code FaceClusteringService} links this region to a {@link Person}
* (new or existing) — {@link EEntityKind#ANIMAL} rows never get one in this iteration, since
* individual animal identity isn't attempted yet, only the species {@link #label}
* @param confirmed set once the future person-management panel confirms this region — never set by automatic
* detection/clustering; see {@code MediaMetadataEditService.updateFaceRegions}, which is the
* only path that ever writes this region out to the file's own XMP packet
*/
@Builder(factoryMethod = "mediaFaceRegion", toBuilder = "from")
public record MediaFaceRegion(@Nullable Long id,
Long mediaFileId,
@Nullable Long personId,
EEntityKind kind,
BoundingBox area,
double confidence,
float @Nullable [] embedding,
@Nullable String label,
String sourceProvider,
boolean confirmed,
Instant detectedAt) {
}
@@ -0,0 +1,21 @@
package org.icroco.pholio.domain.recognition;
import org.jilt.Builder;
import org.jspecify.annotations.Nullable;
import java.time.Instant;
/**
* One recognized identity — a specific person or a specific animal — shared across every
* {@link MediaFaceRegion} that {@code FaceClusteringService} has linked to it.
*
* @param id {@code null} before the row is persisted
* @param name {@code null} until named — either by {@code FaceClusteringService} minting a fresh, unnamed
* cluster, or before the future person-management panel lets a user confirm one
*/
@Builder(factoryMethod = "person", toBuilder = "from")
public record Person(@Nullable Long id,
EEntityKind kind,
@Nullable String name,
Instant createdAt) {
}
@@ -0,0 +1,11 @@
package org.icroco.pholio.domain.recognition;
import java.util.List;
/** What one {@code IRecognitionService.analyze} call found in a single image. */
public record RecognitionResult(List<DetectedRegion> regions) {
public static RecognitionResult empty() {
return new RecognitionResult(List.of());
}
}
@@ -161,7 +161,9 @@ public class MediaAnalysisService {
thumbnailGenerator.decode(target, ImageFormat.JPEG).ifPresentOrElse(
cached -> taskService.execute(TaskType.IMAGE_ANALYSIS, () -> hashAndPublish(mediaFileId, cached, thumbnailBatch)),
() -> {
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId));
// Never assume "unchanged" when the signal that would tell us (a freshly computed
// phash) was never even attempted — see MediaFileAnalyzedEvent's own javadoc.
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId, false));
thumbnailBatch.completedOne();
});
return;
@@ -170,7 +172,7 @@ public class MediaAnalysisService {
Optional<BufferedImage> decoded = thumbnailGenerator.decode(absolute, format.get());
if (decoded.isEmpty()) {
log.debug("No pixels obtainable for '{}'; no thumbnail generated", absolute);
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId));
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId, false));
thumbnailBatch.completedOne();
return;
}
@@ -196,15 +198,23 @@ public class MediaAnalysisService {
}
}
/** Hamming distance at or below which two perceptual hashes are treated as "the same picture". */
private static final int UNCHANGED_PHASH_THRESHOLD = 2;
private void hashAndPublish(Long mediaFileId, BufferedImage thumbnail, TaskService.BatchTask thumbnailBatch) {
try {
long hash = perceptualHasher.phash(thumbnail);
mediaFileRepository.findById(mediaFileId).ifPresent(entity -> {
long hash = perceptualHasher.phash(thumbnail);
boolean unchanged = mediaFileRepository.findById(mediaFileId).map(entity -> {
Long previousHash = entity.getPhash();
entity.setPhash(hash);
entity.setProcessingFlags(MediaFile.withBit(entity.getProcessingFlags(), EMediaFileProcessingFlag.THUMBNAIL_GENERATED));
mediaFileRepository.save(entity);
});
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId));
// A metadata-only rewrite (rating, GPS, capture date) leaves pixels — and so the phash —
// alone; only a genuine content change moves it. See MediaFileAnalyzedEvent's own javadoc
// for why "unknown" (no previous hash) must never be read as "unchanged".
return previousHash != null && PerceptualHasher.hammingDistance(previousHash, hash) <= UNCHANGED_PHASH_THRESHOLD;
}).orElse(false);
publisher.publishEvent(new MediaFileAnalyzedEvent(mediaFileId, unchanged));
}
catch (RuntimeException e) {
log.warn("Perceptual hashing failed unexpectedly: {}", e.toString(), e);
@@ -5,6 +5,14 @@ package org.icroco.pholio.infra.library;
* finishes for one {@link org.icroco.pholio.domain.library.MediaFile} — up to twice per file, since the
* two run independently on their own pools. Listeners are expected to debounce: a large import fires this
* for every file, and nothing needs a UI refresh on each one individually.
*
* @param pixelsLikelyUnchanged {@code true} when the newly computed perceptual hash is close enough to the
* file's previous one that its pixels almost certainly did not change — e.g. a
* metadata-only rewrite (rating, GPS, capture date) touched the file's bytes
* without touching what it shows. {@code false} whenever that cannot be
* determined (new file, no previous hash) or the pixels did change — never
* assumed by default, so a listener that skips expensive re-work on this signal
* (see {@code MediaFileRecognitionTrigger}) only ever does so when genuinely safe.
*/
public record MediaFileAnalyzedEvent(Long mediaFileId) {
public record MediaFileAnalyzedEvent(Long mediaFileId, boolean pixelsLikelyUnchanged) {
}
@@ -0,0 +1,62 @@
package org.icroco.pholio.infra.persistence.recognition;
import lombok.*;
import org.jspecify.annotations.Nullable;
import org.springframework.data.annotation.Id;
import org.springframework.data.relational.core.mapping.Column;
import org.springframework.data.relational.core.mapping.Table;
import java.time.Instant;
/** One detected face/animal bounding box against a {@code media_file} row — see {@link PersonEntity}. */
@Table("media_face_region")
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class MediaFaceRegionEntity {
@Id
@Column("id")
private @Nullable Long id;
@Column("media_file_id")
private Long mediaFileId;
@Column("person_id")
private @Nullable Long personId;
@Column("kind")
private String kind;
@Column("area_x")
private double areaX;
@Column("area_y")
private double areaY;
@Column("area_w")
private double areaW;
@Column("area_h")
private double areaH;
@Column("confidence")
private double confidence;
@Column("embedding")
private byte @Nullable [] embedding;
@Column("label")
private @Nullable String label;
@Column("source_provider")
private String sourceProvider;
@Column("confirmed")
private boolean confirmed;
@Column("detected_at")
private Instant detectedAt;
}
@@ -0,0 +1,36 @@
package org.icroco.pholio.infra.persistence.recognition;
import org.icroco.pholio.domain.recognition.BoundingBox;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.domain.recognition.MediaFaceRegion;
import org.icroco.pholio.infra.recognition.EmbeddingCodec;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
@Mapper(componentModel = "spring", imports = EmbeddingCodec.class)
public interface MediaFaceRegionMapper {
@Mapping(target = "area", expression = "java(toBoundingBox(entity))")
@Mapping(target = "embedding", expression = "java(EmbeddingCodec.fromBytes(entity.getEmbedding()))")
MediaFaceRegion toDomain(MediaFaceRegionEntity entity);
@Mapping(target = "id", ignore = true)
@Mapping(target = "areaX", source = "area.x")
@Mapping(target = "areaY", source = "area.y")
@Mapping(target = "areaW", source = "area.w")
@Mapping(target = "areaH", source = "area.h")
@Mapping(target = "embedding", expression = "java(EmbeddingCodec.toBytes(domain.embedding()))")
MediaFaceRegionEntity toEntity(MediaFaceRegion domain);
default BoundingBox toBoundingBox(MediaFaceRegionEntity entity) {
return new BoundingBox(entity.getAreaX(), entity.getAreaY(), entity.getAreaW(), entity.getAreaH());
}
default String map(EEntityKind kind) {
return kind.name();
}
default EEntityKind map(String kind) {
return EEntityKind.valueOf(kind);
}
}
@@ -0,0 +1,21 @@
package org.icroco.pholio.infra.persistence.recognition;
import java.util.Collection;
import java.util.List;
import org.springframework.data.repository.ListCrudRepository;
public interface MediaFaceRegionRepository extends ListCrudRepository<MediaFaceRegionEntity, Long> {
List<MediaFaceRegionEntity> findByMediaFileId(Long mediaFileId);
List<MediaFaceRegionEntity> findByMediaFileIdIn(Collection<Long> mediaFileIds);
/** Clears a file's previous detections before a re-detection writes its new set — mirrors {@code MediaFileTagRepository}. */
void deleteByMediaFileIdIn(Collection<Long> mediaFileIds);
/** Unclustered faces — what {@code FaceClusteringService} still needs to link to a {@code Person}. */
List<MediaFaceRegionEntity> findByKindAndPersonIdIsNull(String kind);
List<MediaFaceRegionEntity> findByPersonId(Long personId);
}
@@ -0,0 +1,32 @@
package org.icroco.pholio.infra.persistence.recognition;
import lombok.*;
import org.jspecify.annotations.Nullable;
import org.springframework.data.annotation.Id;
import org.springframework.data.relational.core.mapping.Column;
import org.springframework.data.relational.core.mapping.Table;
import java.time.Instant;
/** One recognized identity (a person or an animal) — see {@link MediaFaceRegionEntity} for its detections. */
@Table("person")
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class PersonEntity {
@Id
@Column("id")
private @Nullable Long id;
@Column("kind")
private String kind;
@Column("name")
private @Nullable String name;
@Column("created_at")
private Instant createdAt;
}
@@ -0,0 +1,23 @@
package org.icroco.pholio.infra.persistence.recognition;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.domain.recognition.Person;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
@Mapper(componentModel = "spring")
public interface PersonMapper {
Person toDomain(PersonEntity entity);
@Mapping(target = "id", ignore = true)
PersonEntity toEntity(Person domain);
default String map(EEntityKind kind) {
return kind.name();
}
default EEntityKind map(String kind) {
return EEntityKind.valueOf(kind);
}
}
@@ -0,0 +1,10 @@
package org.icroco.pholio.infra.persistence.recognition;
import org.springframework.data.repository.ListCrudRepository;
import java.util.List;
public interface PersonRepository extends ListCrudRepository<PersonEntity, Long> {
List<PersonEntity> findByKind(String kind);
}
@@ -0,0 +1,18 @@
package org.icroco.pholio.infra.recognition;
/**
* Which wire contract a {@link RecognitionProviderConfig} speaks. Unlike {@code EGeocodingProviderKind},
* there is no well-known third-party face/animal-recognition API to standardise on, so — for now — a remote
* recognition provider is expected to be a small HTTP service speaking Pholio's own contract: {@code POST}
* to {@link RecognitionProviderConfig#urlTemplate()} a JSON body {@code {"apiKey", "width", "height",
* "imageBase64"}} (the decoded image, JPEG-encoded, then base64), and reply with a JSON array of
* {@code {"kind":"PERSON"|"ANIMAL","x","y","w","h","confidence","label"}} (the same normalized {@code
* stArea} center/size convention {@link org.icroco.pholio.domain.recognition.BoundingBox} uses).
*
* <p>Kept as a real enum rather than inlining {@link #HTTP_JSON} everywhere so a second, differently-shaped
* remote contract can be added later as one more case, the same reason {@code EGeocodingProviderKind} does.
*/
public enum ERecognitionProviderKind {
HTTP_JSON
}
@@ -0,0 +1,40 @@
package org.icroco.pholio.infra.recognition;
import org.jspecify.annotations.Nullable;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* {@code float[]} embedding vector &lt;-&gt; the little-endian {@code byte[]} stored in
* {@code media_face_region.embedding} — H2/JDBC has no native vector/array column type, and a plain byte
* buffer needs no third-party (de)serialization library for something this small (typically 128-512 floats).
*/
public final class EmbeddingCodec {
private EmbeddingCodec() {
}
public static byte @Nullable [] toBytes(float @Nullable [] embedding) {
if (embedding == null) {
return null;
}
ByteBuffer buffer = ByteBuffer.allocate(embedding.length * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN);
for (float value : embedding) {
buffer.putFloat(value);
}
return buffer.array();
}
public static float @Nullable [] fromBytes(byte @Nullable [] bytes) {
if (bytes == null) {
return null;
}
ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
float[] embedding = new float[bytes.length / Float.BYTES];
for (int i = 0; i < embedding.length; i++) {
embedding[i] = buffer.getFloat();
}
return embedding;
}
}
@@ -0,0 +1,171 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionEntity;
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionRepository;
import org.icroco.pholio.infra.persistence.recognition.PersonEntity;
import org.icroco.pholio.infra.persistence.recognition.PersonRepository;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.jspecify.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* Groups unnamed {@link EEntityKind#PERSON} face regions into {@link PersonEntity} identities by embedding
* similarity, entirely automatically — no manual step, the "Person 1", "Person 2"... clusters the future
* person-management panel will let a user name. Animal regions are never clustered: individual animal
* identity isn't attempted in this iteration, only the species label already on the region.
*
* <p>Brute-force cosine similarity against one centroid per existing cluster, not a full pairwise
* comparison against every previously-seen face — acceptable at "one photo library" scale (thousands, not
* millions, of faces); see the implementation plan's own risk notes for when that would need revisiting.
*/
@Service
public class FaceClusteringService {
private static final Logger log = LoggerFactory.getLogger(FaceClusteringService.class);
/** SFace's own calibrated same-identity cosine threshold (OpenCV Zoo's face_recognition_sface README). */
private static final double DEFAULT_THRESHOLD = 0.363;
private final MediaFaceRegionRepository regionRepository;
private final PersonRepository personRepository;
private final AppPreferences preferences;
public FaceClusteringService(MediaFaceRegionRepository regionRepository, PersonRepository personRepository,
AppPreferences preferences) {
this.regionRepository = regionRepository;
this.personRepository = personRepository;
this.preferences = preferences;
}
/** Links every unlinked {@link EEntityKind#PERSON} region to an existing or freshly-minted {@link PersonEntity}. */
public void clusterUnnamedPersons() {
List<MediaFaceRegionEntity> unassigned = regionRepository.findByKindAndPersonIdIsNull(EEntityKind.PERSON.name());
if (unassigned.isEmpty()) {
return;
}
double threshold = preferences.getValueOr("recognition", "person-cluster-threshold", Double.class, DEFAULT_THRESHOLD);
List<Cluster> clusters = loadExistingClusters();
List<MediaFaceRegionEntity> toSave = new ArrayList<>();
for (MediaFaceRegionEntity region : unassigned) {
float[] embedding = EmbeddingCodec.fromBytes(region.getEmbedding());
if (embedding == null) {
log.warn("Person region {} has no embedding, cannot cluster it", region.getId());
continue;
}
Cluster best = bestMatch(clusters, embedding, threshold);
if (best == null) {
best = new Cluster(createPerson(), embedding.clone(), 1);
clusters.add(best);
}
else {
best.accumulate(embedding);
}
region.setPersonId(best.personId());
toSave.add(region);
}
if (!toSave.isEmpty()) {
regionRepository.saveAll(toSave);
log.info("Clustered {} face region(s) into {} person(s)", toSave.size(), clusters.size());
}
}
private List<Cluster> loadExistingClusters() {
List<Cluster> clusters = new ArrayList<>();
for (PersonEntity person : personRepository.findByKind(EEntityKind.PERSON.name())) {
Long personId = Objects.requireNonNull(person.getId(), "A persisted PersonEntity always has an id");
List<float[]> embeddings = regionRepository.findByPersonId(personId).stream()
.map(region -> EmbeddingCodec.fromBytes(region.getEmbedding()))
.filter(Objects::nonNull)
.toList();
if (!embeddings.isEmpty()) {
clusters.add(new Cluster(personId, mean(embeddings), embeddings.size()));
}
}
return clusters;
}
private static @Nullable Cluster bestMatch(List<Cluster> clusters, float[] embedding, double threshold) {
Cluster best = null;
double bestScore = threshold;
for (Cluster cluster : clusters) {
double score = cosineSimilarity(cluster.centroid(), embedding);
if (score >= bestScore) {
best = cluster;
bestScore = score;
}
}
return best;
}
private Long createPerson() {
PersonEntity saved = personRepository.save(PersonEntity.builder()
.kind(EEntityKind.PERSON.name())
.name(null)
.createdAt(Instant.now())
.build());
return Objects.requireNonNull(saved.getId(), "A freshly saved PersonEntity always has an id");
}
private static float[] mean(List<float[]> embeddings) {
float[] mean = new float[embeddings.getFirst().length];
for (float[] embedding : embeddings) {
for (int i = 0; i < mean.length; i++) {
mean[i] += embedding[i];
}
}
for (int i = 0; i < mean.length; i++) {
mean[i] /= embeddings.size();
}
return mean;
}
private static double cosineSimilarity(float[] a, float[] b) {
double dot = 0, normA = 0, normB = 0;
for (int i = 0; i < a.length; i++) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
if (normA == 0 || normB == 0) {
return 0;
}
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
}
/** A running centroid — its own field, mutated in place as more regions join it within one clustering pass. */
private static final class Cluster {
private final Long personId;
private final float[] centroid;
private int count;
private Cluster(Long personId, float[] centroid, int count) {
this.personId = personId;
this.centroid = centroid;
this.count = count;
}
private Long personId() {
return personId;
}
private float[] centroid() {
return centroid;
}
private void accumulate(float[] embedding) {
count++;
for (int i = 0; i < centroid.length; i++) {
centroid[i] += (embedding[i] - centroid[i]) / count;
}
}
}
}
@@ -0,0 +1,100 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.domain.recognition.MediaFaceRegion;
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionEntity;
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionMapper;
import org.icroco.pholio.infra.persistence.recognition.MediaFaceRegionRepository;
import org.icroco.pholio.infra.persistence.recognition.PersonEntity;
import org.icroco.pholio.infra.persistence.recognition.PersonRepository;
import org.jspecify.annotations.Nullable;
import org.springframework.stereotype.Service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Reads and replaces a {@code MediaFile}'s detected regions — used by {@link MediaRecognitionService} (writes,
* after each detection pass), {@code MediaInfoPane} (reads, for its persons/animals row) and the future
* person-management panel.
*/
@Service
public class FaceRegionQueryService {
private final MediaFaceRegionRepository repository;
private final MediaFaceRegionMapper mapper;
private final PersonRepository personRepository;
public FaceRegionQueryService(MediaFaceRegionRepository repository, MediaFaceRegionMapper mapper,
PersonRepository personRepository) {
this.repository = repository;
this.mapper = mapper;
this.personRepository = personRepository;
}
public List<MediaFaceRegion> findByMediaFile(Long mediaFileId) {
return repository.findByMediaFileId(mediaFileId).stream().map(mapper::toDomain).toList();
}
/** One entry per detected region, name resolved for {@link EEntityKind#PERSON} rows already linked to a named {@code Person}. */
public List<DisplayRegion> findDisplayRegionsFor(Long mediaFileId) {
List<MediaFaceRegionEntity> regions = repository.findByMediaFileId(mediaFileId);
if (regions.isEmpty()) {
return List.of();
}
Set<Long> personIds = regions.stream().map(MediaFaceRegionEntity::getPersonId).filter(java.util.Objects::nonNull).collect(Collectors.toSet());
Map<Long, String> namesById = personIds.isEmpty() ? Map.of()
: java.util.stream.StreamSupport.stream(personRepository.findAllById(personIds).spliterator(), false)
.filter(person -> person.getName() != null)
.collect(Collectors.toMap(PersonEntity::getId, PersonEntity::getName));
return regions.stream()
.map(region -> new DisplayRegion(EEntityKind.valueOf(region.getKind()), region.getLabel(),
region.getPersonId() == null ? null : namesById.get(region.getPersonId())))
.toList();
}
/** One row for {@code MediaInfoPane}'s persons/animals chips — {@code personName} is {@code null} for an unnamed cluster or any {@link EEntityKind#ANIMAL}. */
public record DisplayRegion(EEntityKind kind, @Nullable String label, @Nullable String personName) {
}
/**
* Replaces every region {@code mediaFileId} previously had with {@code regions} — delete-then-reinsert,
* the same convention {@code LibraryFolderService.persistTags} uses for {@code media_file_tag}, since a
* re-detection's region set is typically small and unrelated row-by-row diffing buys nothing.
*
* <p>Every inserted row starts unlinked ({@code personId = null}) and unconfirmed — see
* {@link MediaFaceRegion#confirmed()}'s own javadoc for why automatic detection never confirms a region
* itself.
*/
public void replaceRegionsFor(Long mediaFileId, List<DetectedRegion> regions) {
repository.deleteByMediaFileIdIn(List.of(mediaFileId));
if (regions.isEmpty()) {
return;
}
Instant now = Instant.now();
List<MediaFaceRegionEntity> entities = regions.stream().map(region -> toEntity(mediaFileId, region, now)).toList();
repository.saveAll(entities);
}
private static MediaFaceRegionEntity toEntity(Long mediaFileId, DetectedRegion region, Instant detectedAt) {
return MediaFaceRegionEntity.builder()
.mediaFileId(mediaFileId)
.personId(null)
.kind(region.kind().name())
.areaX(region.box().x())
.areaY(region.box().y())
.areaW(region.box().w())
.areaH(region.box().h())
.confidence(region.confidence())
.embedding(EmbeddingCodec.toBytes(region.embedding()))
.label(region.label())
.sourceProvider(region.sourceProvider())
.confirmed(false)
.detectedAt(detectedAt)
.build();
}
}
@@ -0,0 +1,18 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.recognition.RecognitionResult;
import java.awt.image.BufferedImage;
/**
* Finds faces and animals in one already-decoded photo — the local/remote provider seam, mirroring
* {@code IPlaceSearchService}. {@link LocalRecognitionService} answers entirely offline, delegating to
* whichever {@code engine} package implementation is wired (DJL/ONNX today, swappable later without
* touching this seam); {@link RecognitionService} is the {@code @Primary} bean actually injected everywhere,
* routing each call to either the local engines or a user-configured remote provider (see its own javadoc)
* so the caller never needs to know which one actually answered.
*/
public interface IRecognitionService {
RecognitionResult analyze(BufferedImage image);
}
@@ -0,0 +1,35 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import org.icroco.pholio.domain.recognition.RecognitionResult;
import org.icroco.pholio.infra.recognition.engine.IFaceDetectionEngine;
import org.icroco.pholio.infra.recognition.engine.IObjectDetectionEngine;
import org.springframework.stereotype.Component;
import java.awt.image.BufferedImage;
import java.util.ArrayList;
import java.util.List;
/**
* {@link IRecognitionService} answered entirely by the local engines — no network involved. The class name
* itself is the "runs locally" signal {@link RecognitionService}'s own javadoc refers to, the same
* convention {@code LocalPlaceSearchService} uses for geocoding.
*/
@Component
public class LocalRecognitionService implements IRecognitionService {
private final IFaceDetectionEngine faceEngine;
private final IObjectDetectionEngine animalEngine;
public LocalRecognitionService(IFaceDetectionEngine faceEngine, IObjectDetectionEngine animalEngine) {
this.faceEngine = faceEngine;
this.animalEngine = animalEngine;
}
@Override
public RecognitionResult analyze(BufferedImage image) {
List<DetectedRegion> regions = new ArrayList<>(faceEngine.detectFaces(image));
regions.addAll(animalEngine.detectAnimals(image));
return new RecognitionResult(regions);
}
}
@@ -0,0 +1,88 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
import org.icroco.pholio.infra.library.LibraryFolderService;
import org.icroco.pholio.infra.persistence.folder.MediaFileEntity;
import org.icroco.pholio.infra.persistence.folder.MediaFileMapper;
import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.icroco.pholio.infra.scheduling.IStartupTask;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.infra.task.TaskType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.DependsOn;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Objects;
/**
* Backfills face/animal detection, at every application start, for every {@code media_file} row in the
* currently open library still missing {@link EMediaFileProcessingFlag#FACE_DETECTED} — the library-wide
* catch-up counterpart to {@link MediaFileRecognitionTrigger}'s per-import/per-sync hook, for files that
* predate this feature or whose earlier detection attempt never completed.
*
* <p>Unlike {@code MediaFileProcessingFlagsBackfillTask} (a cheap flag-only fix), this dispatches real
* detection work, so it is submitted as a single, <em>visible</em> {@code TaskType.IMAGE_ANALYSIS} batch
* (not silent) — the "N files remaining" progress this feature is explicitly meant to show in the status
* bar, rather than a silent background fix.
*
* <p>{@code @Order(1)}: runs after {@code MediaFileProcessingFlagsBackfillTask} (@Order(0)), so this reads
* flags that backfill has already reconciled with reality.
*
* <p>{@code @DependsOn("libraryService")} for the same reason as that task: {@code media_file} lives in the
* per-library routing datasource, which must already point at an open library.
*/
@Component
@DependsOn("libraryService")
@Order(1)
public class MediaFaceDetectionBackfillTask implements IStartupTask {
private static final Logger log = LoggerFactory.getLogger(MediaFaceDetectionBackfillTask.class);
private final AppPreferences preferences;
private final MediaFileRepository mediaFileRepository;
private final MediaFileMapper mediaFileMapper;
private final LibraryFolderService libraryFolderService;
private final MediaRecognitionService mediaRecognitionService;
private final TaskService taskService;
public MediaFaceDetectionBackfillTask(AppPreferences preferences, MediaFileRepository mediaFileRepository,
MediaFileMapper mediaFileMapper, LibraryFolderService libraryFolderService,
MediaRecognitionService mediaRecognitionService, TaskService taskService) {
this.preferences = preferences;
this.mediaFileRepository = mediaFileRepository;
this.mediaFileMapper = mediaFileMapper;
this.libraryFolderService = libraryFolderService;
this.mediaRecognitionService = mediaRecognitionService;
this.taskService = taskService;
}
@Override
public boolean shouldRun() {
return preferences.getValueOr("recognition", "enabled", Boolean.class, true);
}
@Override
public void run() {
List<MediaFileEntity> pending = mediaFileRepository.findAll().stream()
.filter(entity -> !mediaFileMapper.toDomain(entity)
.hasProcessingFlag(EMediaFileProcessingFlag.FACE_DETECTED))
.toList();
if (pending.isEmpty()) {
log.debug("No media file pending face/animal detection");
return;
}
TaskService.BatchTask batch = taskService.submitBatch(TaskType.IMAGE_ANALYSIS, "Detecting faces & animals", pending.size(), false);
pending.forEach(entity -> {
Long mediaFileId = Objects.requireNonNull(entity.getId(), "A persisted MediaFileEntity always has an id");
libraryFolderService.absolutePathOf(mediaFileMapper.toDomain(entity))
.ifPresentOrElse(
absolute -> taskService.execute(TaskType.IMAGE_ANALYSIS,
() -> mediaRecognitionService.detect(mediaFileId, absolute, batch)),
batch::completedOne);
});
}
}
@@ -0,0 +1,65 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
import org.icroco.pholio.domain.library.MediaFile;
import org.icroco.pholio.infra.library.LibraryFolderService;
import org.icroco.pholio.infra.library.MediaFileAnalyzedEvent;
import org.icroco.pholio.infra.persistence.folder.MediaFileMapper;
import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.infra.task.TaskType;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import java.util.Objects;
/**
* Runs face/animal detection for a media file right after {@code MediaAnalysisService} finishes its own
* thumbnail+phash pass — covers both a freshly imported file and a re-synced, modified one, since
* {@code LibraryFolderService.reimportModifiedFile} already calls {@code generateThumbnail} unconditionally,
* whose {@link MediaFileAnalyzedEvent} this listens to either way. No separate hook into
* {@code SyncReportService} is needed.
*
* <p>Skips already-detected files whose pixels are unchanged ({@link MediaFileAnalyzedEvent#pixelsLikelyUnchanged()}
* — a metadata-only edit, e.g. a rating change, re-fires this event but must not re-run detection.
*/
@Component
public class MediaFileRecognitionTrigger {
private final AppPreferences preferences;
private final MediaFileRepository mediaFileRepository;
private final MediaFileMapper mediaFileMapper;
private final LibraryFolderService libraryFolderService;
private final MediaRecognitionService mediaRecognitionService;
private final TaskService taskService;
public MediaFileRecognitionTrigger(AppPreferences preferences, MediaFileRepository mediaFileRepository,
MediaFileMapper mediaFileMapper, LibraryFolderService libraryFolderService,
MediaRecognitionService mediaRecognitionService, TaskService taskService) {
this.preferences = preferences;
this.mediaFileRepository = mediaFileRepository;
this.mediaFileMapper = mediaFileMapper;
this.libraryFolderService = libraryFolderService;
this.mediaRecognitionService = mediaRecognitionService;
this.taskService = taskService;
}
@EventListener
public void onAnalyzed(MediaFileAnalyzedEvent event) {
if (!preferences.getValueOr("recognition", "enabled", Boolean.class, true)) {
return;
}
mediaFileRepository.findById(event.mediaFileId()).ifPresent(entity -> {
MediaFile mediaFile = mediaFileMapper.toDomain(entity);
if (mediaFile.hasProcessingFlag(EMediaFileProcessingFlag.FACE_DETECTED) && event.pixelsLikelyUnchanged()) {
return;
}
Long mediaFileId = Objects.requireNonNull(entity.getId(), "A persisted MediaFileEntity always has an id");
libraryFolderService.absolutePathOf(mediaFile).ifPresent(absolute -> {
TaskService.BatchTask unit = taskService.submitBatch(TaskType.IMAGE_ANALYSIS, "Detecting faces & animals", 1, true);
taskService.execute(TaskType.IMAGE_ANALYSIS, () -> mediaRecognitionService.detect(mediaFileId, absolute, unit));
});
});
}
}
@@ -0,0 +1,9 @@
package org.icroco.pholio.infra.recognition;
/**
* Published once {@code MediaRecognitionService.detect} has replaced a {@code MediaFile}'s detected
* regions — {@code MediaInfoPane}'s persons/animals row listens for this to refresh if the file it is
* currently showing is the one just analyzed.
*/
public record MediaFileRecognizedEvent(Long mediaFileId) {
}
@@ -0,0 +1,95 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.library.EMediaFileProcessingFlag;
import org.icroco.pholio.domain.library.MediaFile;
import org.icroco.pholio.domain.media.ImageFormat;
import org.icroco.pholio.domain.recognition.RecognitionResult;
import org.icroco.pholio.infra.media.MediaFormatRegistry;
import org.icroco.pholio.infra.media.ThumbnailGenerator;
import org.icroco.pholio.infra.persistence.folder.MediaFileRepository;
import org.icroco.pholio.infra.task.TaskService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Service;
import java.awt.image.BufferedImage;
import java.nio.file.Path;
import java.util.Optional;
/**
* The per-file face/animal detection pipeline — decode, analyze, persist, cluster, flag, notify. Run off
* {@code TaskType.IMAGE_ANALYSIS} by both {@link MediaFileRecognitionTrigger} (import/re-sync) and
* {@link MediaFaceDetectionBackfillTask} (startup backlog).
*
* <p>Decodes the original file rather than reusing the small cached thumbnail, for accuracy on small/distant
* faces — more expensive than the thumbnail path {@code MediaAnalysisService} optimizes for reuse; see the
* implementation plan's own risk notes if this needs revisiting against very large libraries.
*
* <p>{@code @DependsOn("libraryService")} for the same reason as {@code MediaAnalysisService}: the routing
* datasource must already point at an open library before this touches a repository.
*/
@Service
@DependsOn("libraryService")
public class MediaRecognitionService {
private static final Logger log = LoggerFactory.getLogger(MediaRecognitionService.class);
private final MediaFormatRegistry formats;
private final ThumbnailGenerator thumbnailGenerator;
private final IRecognitionService recognitionService;
private final FaceRegionQueryService faceRegionQueryService;
private final FaceClusteringService faceClusteringService;
private final MediaFileRepository mediaFileRepository;
private final ApplicationEventPublisher publisher;
public MediaRecognitionService(MediaFormatRegistry formats, ThumbnailGenerator thumbnailGenerator,
IRecognitionService recognitionService, FaceRegionQueryService faceRegionQueryService,
FaceClusteringService faceClusteringService, MediaFileRepository mediaFileRepository,
ApplicationEventPublisher publisher) {
this.formats = formats;
this.thumbnailGenerator = thumbnailGenerator;
this.recognitionService = recognitionService;
this.faceRegionQueryService = faceRegionQueryService;
this.faceClusteringService = faceClusteringService;
this.mediaFileRepository = mediaFileRepository;
this.publisher = publisher;
}
/**
* @param batchTask completed exactly once, in a {@code finally}, whatever the outcome — the same
* discipline {@code MediaAnalysisService.hashAndPublish} follows for its own batch.
*/
public void detect(Long mediaFileId, Path absolute, TaskService.BatchTask batchTask) {
try {
Optional<ImageFormat> format = formats.formatOf(absolute);
if (format.isEmpty()) {
log.warn("'{}' is no longer a recognised format; skipping recognition", absolute);
return;
}
Optional<BufferedImage> decoded = thumbnailGenerator.decode(absolute, format.get());
if (decoded.isEmpty()) {
log.debug("No pixels obtainable for '{}'; no recognition run", absolute);
return;
}
BufferedImage oriented = thumbnailGenerator.applyOrientation(decoded.get(), thumbnailGenerator.orientationOf(absolute));
RecognitionResult result = recognitionService.analyze(oriented);
faceRegionQueryService.replaceRegionsFor(mediaFileId, result.regions());
faceClusteringService.clusterUnnamedPersons();
mediaFileRepository.findById(mediaFileId).ifPresent(entity -> {
entity.setProcessingFlags(MediaFile.withBit(entity.getProcessingFlags(), EMediaFileProcessingFlag.FACE_DETECTED));
mediaFileRepository.save(entity);
});
publisher.publishEvent(new MediaFileRecognizedEvent(mediaFileId));
}
catch (RuntimeException e) {
log.warn("Face/animal recognition failed unexpectedly for '{}': {}", absolute, e.toString(), e);
}
finally {
batchTask.completedOne();
}
}
}
@@ -0,0 +1,15 @@
package org.icroco.pholio.infra.recognition;
/**
* One user-configured remote recognition provider, as stored (a JSON list of these) in the
* {@code recognition.providers-json} preference — see {@link RecognitionService} for how it is picked and
* used, and {@link ERecognitionProviderKind} for the wire contract it must speak.
*
* @param name shown in the "active provider" picker; also the key {@code recognition.active-provider}
* stores to select this config
* @param urlTemplate the endpoint to {@code POST} the analysis request to
* @param apiKey sent as-is in the request body; the provider validates it however it wants
* @param kind which wire contract to speak — see {@link ERecognitionProviderKind}
*/
public record RecognitionProviderConfig(String name, String urlTemplate, String apiKey, ERecognitionProviderKind kind) {
}
@@ -0,0 +1,181 @@
package org.icroco.pholio.infra.recognition;
import org.icroco.pholio.domain.recognition.BoundingBox;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.domain.recognition.RecognitionResult;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import tools.jackson.core.type.TypeReference;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.node.ObjectNode;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
/**
* The {@code @Primary} {@link IRecognitionService} — the one bean actually injected wherever face/animal
* recognition is needed. Routes each call to either {@link LocalRecognitionService} or a user-configured
* remote provider, transparently to the caller — the exact same shape {@code PlaceSearchService} uses for
* geocoding, just a strict proxy rather than a merge: a remote provider's answer is used as-is, never
* combined with the local engines' own.
*
* <ul>
* <li>{@code recognition.active-provider} blank/unset (the default) &rarr; {@link LocalRecognitionService},
* always.
* <li>otherwise the matching {@link RecognitionProviderConfig} from {@code recognition.providers-json} is
* called over HTTP, per {@link ERecognitionProviderKind}'s wire contract.
* </ul>
*
* <p>Falls back to {@link LocalRecognitionService} whenever the configured provider cannot answer — an
* unknown/deleted provider name, a non-2xx response, a network failure, a malformed reply — rather than
* surfacing an error: local detection is always available, so the background pipeline keeps working offline
* the same way it always has.
*/
@Component
@Primary
public class RecognitionService implements IRecognitionService {
private static final Logger log = LoggerFactory.getLogger(RecognitionService.class);
private static final String PREFERENCE_GROUP = "recognition";
private static final String ACTIVE_PROVIDER_KEY = "active-provider";
private static final String PROVIDERS_KEY = "providers-json";
private final LocalRecognitionService local;
private final AppPreferences preferences;
private final ObjectMapper json = new ObjectMapper();
private final HttpClient httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(5))
.build();
public RecognitionService(LocalRecognitionService local, AppPreferences preferences) {
this.local = local;
this.preferences = preferences;
}
@Override
public RecognitionResult analyze(BufferedImage image) {
String activeProviderName = preferences.text(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY).orElse("");
if (activeProviderName.isBlank()) {
log.debug("No active recognition provider configured, analyzing locally");
return local.analyze(image);
}
log.debug("Active recognition provider is '{}', analyzing remotely", activeProviderName);
return providers().stream()
.filter(provider -> provider.name().equals(activeProviderName))
.findFirst()
.map(provider -> analyzeRemote(provider, image))
.orElseGet(() -> {
log.warn("Active recognition provider '{}' is no longer configured, falling back to local analysis",
activeProviderName);
return local.analyze(image);
});
}
/** The configured provider list, in the order they were added — never {@code null}. Exposed for the Maintenance recognition tab. */
public List<RecognitionProviderConfig> providers() {
String raw = preferences.text(PREFERENCE_GROUP, PROVIDERS_KEY).orElse("[]");
try {
return json.readValue(raw, new TypeReference<List<RecognitionProviderConfig>>() {});
}
catch (RuntimeException e) {
log.warn("Could not read recognition.providers-json ('{}') as a provider list, treating it as empty", raw, e);
return List.of();
}
}
/** Writes {@code providers} back to {@code recognition.providers-json} — the Maintenance tab's Save action. */
public void saveProviders(List<RecognitionProviderConfig> providers) {
preferences.setValue(PREFERENCE_GROUP, PROVIDERS_KEY, json.writeValueAsString(providers));
}
/** The active provider's name, or blank for "local only". */
public String activeProvider() {
return preferences.text(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY).orElse("");
}
/** Writes {@code providerName} (blank for "local only") to {@code recognition.active-provider}. */
public void saveActiveProvider(String providerName) {
preferences.setValue(PREFERENCE_GROUP, ACTIVE_PROVIDER_KEY, providerName);
}
private RecognitionResult analyzeRemote(RecognitionProviderConfig provider, BufferedImage image) {
log.info("Calling recognition provider '{}' (kind={})", provider.name(), provider.kind());
try {
String body = requestBody(provider, image);
HttpRequest request = HttpRequest.newBuilder(URI.create(provider.urlTemplate()))
.header("Content-Type", "application/json")
.timeout(Duration.ofSeconds(20))
.POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8))
.build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
log.info("Recognition provider '{}' responded HTTP {}", provider.name(), response.statusCode());
if (response.statusCode() != 200) {
log.warn("Recognition provider '{}' returned HTTP {}, falling back to local analysis",
provider.name(), response.statusCode());
return local.analyze(image);
}
RecognitionResult result = parse(response.body(), provider.name());
log.info("Recognition provider '{}' returned {} region(s)", provider.name(), result.regions().size());
return result;
}
catch (IOException e) {
log.warn("Could not reach recognition provider '{}', falling back to local analysis", provider.name(), e);
return local.analyze(image);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
return local.analyze(image);
}
catch (RuntimeException e) {
// A remote service changing shape, or simply misbehaving, must not crash the recognition pipeline.
log.warn("Could not parse recognition provider '{}' response, falling back to local analysis", provider.name(), e);
return local.analyze(image);
}
}
private String requestBody(RecognitionProviderConfig provider, BufferedImage image) throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", buffer);
ObjectNode node = json.createObjectNode();
node.put("apiKey", provider.apiKey());
node.put("width", image.getWidth());
node.put("height", image.getHeight());
node.put("imageBase64", Base64.getEncoder().encodeToString(buffer.toByteArray()));
return json.writeValueAsString(node);
}
private RecognitionResult parse(String body, String providerName) {
JsonNode root = json.readTree(body);
List<DetectedRegion> regions = new ArrayList<>();
for (JsonNode node : root) {
regions.add(toDetectedRegion(node, providerName));
}
return new RecognitionResult(regions);
}
private static DetectedRegion toDetectedRegion(JsonNode node, String providerName) {
EEntityKind kind = EEntityKind.valueOf(node.path("kind").asString("PERSON"));
BoundingBox box = new BoundingBox(node.path("x").asDouble(), node.path("y").asDouble(),
node.path("w").asDouble(), node.path("h").asDouble());
return new DetectedRegion(kind, box, node.path("confidence").asDouble(1.0),
null, node.path("label").asString(null), providerName);
}
}
@@ -0,0 +1,17 @@
package org.icroco.pholio.infra.recognition.engine;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* The local face-detection/embedding engine seam — independent of {@code IRecognitionService}'s own
* local-vs-remote seam, so swapping this (DJL/ONNX today, JavaCV/OpenCV DNN or anything else later) never
* touches {@code RecognitionService}'s routing logic. Each returned {@link DetectedRegion} carries an
* embedding for {@code FaceClusteringService} to group across photos.
*/
public interface IFaceDetectionEngine {
List<DetectedRegion> detectFaces(BufferedImage image);
}
@@ -0,0 +1,21 @@
package org.icroco.pholio.infra.recognition.engine;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* The local animal-detection engine seam — see {@link IFaceDetectionEngine}'s own javadoc for why this is
* kept separate from {@code IRecognitionService}'s local-vs-remote seam. Each returned {@link DetectedRegion}
* carries a species {@code label} (e.g. {@code "dog"}), not an embedding — individual animal identity is not
* attempted in this iteration.
*
* <p>Named for animals specifically rather than "objects" in general: generic object recognition is a
* separate, later phase, and will get its own method here (or a sibling interface) once designed, without
* needing to touch any existing caller.
*/
public interface IObjectDetectionEngine {
List<DetectedRegion> detectAnimals(BufferedImage image);
}
@@ -0,0 +1,15 @@
package org.icroco.pholio.infra.recognition.engine;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import java.awt.image.BufferedImage;
import java.util.List;
/** Placeholder {@link IObjectDetectionEngine} — see {@link NoopFaceDetectionEngine}'s own javadoc. */
public class NoopAnimalDetectionEngine implements IObjectDetectionEngine {
@Override
public List<DetectedRegion> detectAnimals(BufferedImage image) {
return List.of();
}
}
@@ -0,0 +1,19 @@
package org.icroco.pholio.infra.recognition.engine;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import java.awt.image.BufferedImage;
import java.util.List;
/**
* Placeholder {@link IFaceDetectionEngine} — always finds nothing. Superseded by
* {@code YuNetSFaceFaceDetectionEngine} as the real {@code @Component}; kept as a plain (non-Spring) class
* for tests that need a face engine stand-in without loading actual ONNX models.
*/
public class NoopFaceDetectionEngine implements IFaceDetectionEngine {
@Override
public List<DetectedRegion> detectFaces(BufferedImage image) {
return List.of();
}
}
@@ -0,0 +1,41 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
/** Greedy IoU-based NMS, shared by every detector in this package — no per-class grouping (callers that need it group first). */
final class NonMaxSuppression {
private NonMaxSuppression() {
}
record Box(double x1, double y1, double x2, double y2, double score, int index) {
}
/** {@code box.index()} of every survivor, highest score first. */
static List<Integer> suppress(List<Box> boxes, double iouThreshold) {
List<Box> byScoreDesc = boxes.stream().sorted(Comparator.comparingDouble(Box::score).reversed()).toList();
List<Box> kept = new ArrayList<>();
List<Integer> result = new ArrayList<>();
for (Box candidate : byScoreDesc) {
boolean overlapsKept = kept.stream().anyMatch(k -> iou(candidate, k) > iouThreshold);
if (!overlapsKept) {
kept.add(candidate);
result.add(candidate.index());
}
}
return result;
}
private static double iou(Box a, Box b) {
double x1 = Math.max(a.x1(), b.x1());
double y1 = Math.max(a.y1(), b.y1());
double x2 = Math.min(a.x2(), b.x2());
double y2 = Math.min(a.y2(), b.y2());
double inter = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
double areaA = (a.x2() - a.x1()) * (a.y2() - a.y1());
double areaB = (b.x2() - b.x1()) * (b.y2() - b.y1());
return inter / (areaA + areaB - inter + 1e-9);
}
}
@@ -0,0 +1,47 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import java.awt.image.BufferedImage;
/** NCHW float tensor extraction from a {@link BufferedImage} — shared by every engine in this package. */
final class OnnxImageTensors {
private OnnxImageTensors() {
}
/**
* Raw (unnormalized, 0-255) pixel values in planar NCHW layout — every model bundled here was exported
* from OpenCV/PyTorch pipelines that feed raw pixel floats, never {@code /255}-scaled ones.
*
* @param swapToBgr {@code true} to write the R/G/B planes in B,G,R order — every bundled model here was
* trained against OpenCV's native BGR channel order except SFace, which explicitly
* swaps to RGB before its own forward pass (see {@code YuNetSFaceFaceDetectionEngine}).
* {@link BufferedImage#getRGB} always hands back R/G/B regardless of the source file's
* own encoding, so this is the one place that channel order is ever chosen.
*/
static float[] toChwFloats(BufferedImage image, boolean swapToBgr) {
int width = image.getWidth();
int height = image.getHeight();
int plane = width * height;
float[] data = new float[3 * plane];
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int rgb = image.getRGB(x, y);
int r = (rgb >> 16) & 0xFF;
int g = (rgb >> 8) & 0xFF;
int b = rgb & 0xFF;
int idx = y * width + x;
if (swapToBgr) {
data[idx] = b;
data[plane + idx] = g;
data[2 * plane + idx] = r;
}
else {
data[idx] = r;
data[plane + idx] = g;
data[2 * plane + idx] = b;
}
}
}
return data;
}
}
@@ -0,0 +1,59 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
/**
* Opens an {@link OrtSession} for a bundled ONNX model — packaged as a classpath resource under
* {@code /models/recognition/}, since ONNX Runtime's Java API needs a real file path (or byte array; a file
* path is what lets ONNX Runtime memory-map the weights instead of holding a second copy in the JVM heap).
* The resource is extracted to the OS temp directory once and reused on every later call/run — its filename
* alone is the cache key, since these bundled models never change without a Pholio version bump.
*/
final class OnnxModelLoader {
private static final OrtEnvironment ENVIRONMENT = OrtEnvironment.getEnvironment();
private OnnxModelLoader() {
}
static OrtSession load(String classpathResource) {
try {
Path modelFile = extractToCache(classpathResource);
return ENVIRONMENT.createSession(modelFile.toString(), new OrtSession.SessionOptions());
}
catch (IOException e) {
throw new UncheckedIOException("Could not extract bundled ONNX model '" + classpathResource + "'", e);
}
catch (OrtException e) {
throw new IllegalStateException("Could not open ONNX session for '" + classpathResource + "'", e);
}
}
private static Path extractToCache(String classpathResource) throws IOException {
Path cacheDir = Path.of(System.getProperty("java.io.tmpdir"), "pholio-recognition-models");
Files.createDirectories(cacheDir);
String fileName = classpathResource.substring(classpathResource.lastIndexOf('/') + 1);
Path target = cacheDir.resolve(fileName);
if (Files.exists(target)) {
return target;
}
try (InputStream in = OnnxModelLoader.class.getResourceAsStream(classpathResource)) {
if (in == null) {
throw new IOException("Missing bundled model resource: " + classpathResource);
}
Path staging = Files.createTempFile(cacheDir, "extract-", ".onnx");
Files.copy(in, staging, StandardCopyOption.REPLACE_EXISTING);
Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING);
}
return target;
}
}
@@ -0,0 +1,48 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
/**
* The least-squares 2D similarity transform (uniform scale + rotation + translation, no shear/reflection)
* mapping {@code src} points onto {@code dst} points — used to align a detected face's 5 landmarks onto
* SFace's fixed reference layout before cropping. Equivalent to Umeyama's algorithm restricted to the
* pure-similarity case, but solved directly as a complex-number linear regression rather than via an SVD:
* treating each 2D point as a complex number {@code p = x + iy}, the best-fit {@code dst ≈ a*src + t} (a, t
* complex) minimizing squared error has the closed form below — no matrix decomposition needed.
*/
final class SimilarityTransform {
private SimilarityTransform() {
}
/** {@code {aRe, aIm, tRe, tIm}} such that {@code dst ≈ a*src + t} with {@code a = aRe + i*aIm}, {@code t = tRe + i*tIm}. */
static double[] estimate(double[][] src, double[][] dst) {
int n = src.length;
double meanSrcX = 0, meanSrcY = 0, meanDstX = 0, meanDstY = 0;
for (int i = 0; i < n; i++) {
meanSrcX += src[i][0];
meanSrcY += src[i][1];
meanDstX += dst[i][0];
meanDstY += dst[i][1];
}
meanSrcX /= n;
meanSrcY /= n;
meanDstX /= n;
meanDstY /= n;
double numRe = 0, numIm = 0, den = 0;
for (int i = 0; i < n; i++) {
double sx = src[i][0] - meanSrcX;
double sy = src[i][1] - meanSrcY;
double dx = dst[i][0] - meanDstX;
double dy = dst[i][1] - meanDstY;
// conj(s) * d = (sx - i*sy)(dx + i*dy) = (sx*dx + sy*dy) + i*(sx*dy - sy*dx)
numRe += sx * dx + sy * dy;
numIm += sx * dy - sy * dx;
den += sx * sx + sy * sy;
}
double aRe = numRe / den;
double aIm = numIm / den;
double tRe = meanDstX - (aRe * meanSrcX - aIm * meanSrcY);
double tIm = meanDstY - (aIm * meanSrcX + aRe * meanSrcY);
return new double[]{ aRe, aIm, tRe, tIm };
}
}
@@ -0,0 +1,184 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
import jakarta.annotation.PreDestroy;
import org.icroco.pholio.domain.recognition.BoundingBox;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.infra.recognition.engine.IObjectDetectionEngine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.RenderingHints;
import java.awt.image.BufferedImage;
import java.nio.FloatBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* {@link IObjectDetectionEngine} backed by <a href="https://github.com/Megvii-BaseDetection/YOLOX">YOLOX-Nano</a>
* (Apache-2.0, Megvii), a general 80-class COCO detector filtered down to the animal classes — species label
* only, no individual animal identity in this iteration. Preprocessing (letterbox, gray padding) and
* postprocessing (grid/stride decode, sigmoid already baked into the exported graph) follow YOLOX's own
* {@code demo/ONNXRuntime/onnx_inference.py} and {@code yolox/data/data_augment.py} exactly; input/output
* tensor names and shape were confirmed directly against the bundled {@code .onnx} file via
* {@code OrtSession.getInputInfo()}/{@code getOutputInfo()}.
*/
@Component
public class YoloXAnimalDetectionEngine implements IObjectDetectionEngine, AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(YoloXAnimalDetectionEngine.class);
private static final String SOURCE_PROVIDER = "local-onnx";
private static final int INPUT_SIZE = 416;
private static final int[] STRIDES = { 8, 16, 32 };
/** YOLOX-Nano's own {@code demo/ONNXRuntime} defaults. */
private static final double SCORE_THRESHOLD = 0.3;
private static final double NMS_THRESHOLD = 0.45;
/** COCO's 80 class names, official order — only indices {@link #ANIMAL_CLASS_MIN}..{@link #ANIMAL_CLASS_MAX} are ever looked at. */
private static final String[] COCO_CLASSES = {
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
"fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
"elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
"skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
"tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
"sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
"potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
"microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
"hair drier", "toothbrush"
};
private static final int ANIMAL_CLASS_MIN = 14; // bird
private static final int ANIMAL_CLASS_MAX = 23; // giraffe
private final OrtEnvironment environment = OrtEnvironment.getEnvironment();
private final OrtSession session;
public YoloXAnimalDetectionEngine() {
this.session = OnnxModelLoader.load("/models/recognition/yolox_nano.onnx");
}
@Override
public List<DetectedRegion> detectAnimals(BufferedImage image) {
try {
List<RawDetection> detections = detectRaw(image);
List<DetectedRegion> regions = new ArrayList<>(detections.size());
for (RawDetection d : detections) {
double boxWidth = d.x2() - d.x1();
double boxHeight = d.y2() - d.y1();
BoundingBox box = new BoundingBox((d.x1() + boxWidth / 2) / image.getWidth(),
(d.y1() + boxHeight / 2) / image.getHeight(),
boxWidth / image.getWidth(), boxHeight / image.getHeight());
regions.add(new DetectedRegion(EEntityKind.ANIMAL, box, d.score(), null, d.label(), SOURCE_PROVIDER));
}
return regions;
}
catch (OrtException e) {
log.warn("Animal detection failed unexpectedly: {}", e.toString(), e);
return List.of();
}
}
private record RawDetection(double x1, double y1, double x2, double y2, double score, String label) {
}
private List<RawDetection> detectRaw(BufferedImage image) throws OrtException {
Letterbox letterbox = letterbox(image, INPUT_SIZE);
float[] chw = OnnxImageTensors.toChwFloats(letterbox.image(), true); // YOLOX expects BGR, raw 0-255 (no /255 normalization)
List<RawDetection> candidates;
try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, INPUT_SIZE, INPUT_SIZE });
OrtSession.Result result = session.run(Map.of("images", input))) {
float[][][] output = (float[][][]) result.get("output").orElseThrow().getValue();
candidates = decode(output[0]);
}
List<NonMaxSuppression.Box> boxes = new ArrayList<>(candidates.size());
for (int i = 0; i < candidates.size(); i++) {
RawDetection c = candidates.get(i);
boxes.add(new NonMaxSuppression.Box(c.x1(), c.y1(), c.x2(), c.y2(), c.score(), i));
}
List<Integer> kept = NonMaxSuppression.suppress(boxes, NMS_THRESHOLD);
List<RawDetection> scaled = new ArrayList<>(kept.size());
for (int index : kept) {
RawDetection c = candidates.get(index);
scaled.add(new RawDetection(c.x1() / letterbox.ratio(), c.y1() / letterbox.ratio(),
c.x2() / letterbox.ratio(), c.y2() / letterbox.ratio(), c.score(), c.label()));
}
return scaled;
}
/** {@code predictions}: {@code [numAnchors][85]} — 4 box + 1 objectness + 80 class scores, sigmoid already applied in-graph. */
private static List<RawDetection> decode(float[][] predictions) {
List<RawDetection> out = new ArrayList<>();
int offset = 0;
for (int stride : STRIDES) {
int side = INPUT_SIZE / stride;
for (int gy = 0; gy < side; gy++) {
for (int gx = 0; gx < side; gx++) {
float[] pred = predictions[offset + gy * side + gx];
double cx = (pred[0] + gx) * stride;
double cy = (pred[1] + gy) * stride;
double w = Math.exp(pred[2]) * stride;
double h = Math.exp(pred[3]) * stride;
double obj = pred[4];
int bestClass = -1;
double bestScore = 0;
for (int c = ANIMAL_CLASS_MIN; c <= ANIMAL_CLASS_MAX; c++) {
double classScore = pred[5 + c];
if (classScore > bestScore) {
bestScore = classScore;
bestClass = c;
}
}
double score = obj * bestScore;
if (score < SCORE_THRESHOLD) {
continue;
}
out.add(new RawDetection(cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2, score, COCO_CLASSES[bestClass]));
}
}
offset += side * side;
}
return out;
}
private record Letterbox(BufferedImage image, double ratio) {
}
/** Resizes preserving aspect ratio onto a {@code target}×{@code target} gray (114,114,114) canvas, content anchored top-left — YOLOX's own {@code preproc}. */
private static Letterbox letterbox(BufferedImage source, int target) {
double ratio = Math.min(target / (double) source.getHeight(), target / (double) source.getWidth());
int newW = Math.round((float) (source.getWidth() * ratio));
int newH = Math.round((float) (source.getHeight() * ratio));
BufferedImage canvas = new BufferedImage(target, target, BufferedImage.TYPE_INT_RGB);
Graphics2D g = canvas.createGraphics();
try {
g.setColor(new Color(114, 114, 114));
g.fillRect(0, 0, target, target);
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
g.drawImage(source, 0, 0, newW, newH, null);
}
finally {
g.dispose();
}
return new Letterbox(canvas, ratio);
}
@Override
@PreDestroy
public void close() throws OrtException {
session.close();
}
}
@@ -0,0 +1,233 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
import jakarta.annotation.PreDestroy;
import org.icroco.pholio.domain.recognition.BoundingBox;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.infra.recognition.engine.IFaceDetectionEngine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.awt.Graphics2D;
import java.awt.RenderingHints;
import java.awt.geom.AffineTransform;
import java.awt.image.BufferedImage;
import java.nio.FloatBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* {@link IFaceDetectionEngine} backed by two bundled ONNX models, chained: <a href="https://github.com/opencv/opencv_zoo/tree/main/models/face_detection_yunet">YuNet</a>
* (bounding box + 5 landmarks) then <a href="https://github.com/opencv/opencv_zoo/tree/main/models/face_recognition_sface">SFace</a>
* (128-d embedding, run on a landmark-aligned 112×112 crop) — both from OpenCV Zoo, MIT/Apache-2.0 licensed.
* Every input tensor shape, output tensor name, decode formula and reference landmark constant below was
* either taken verbatim from OpenCV's own {@code face_detect.cpp}/{@code face_recognize.cpp} (the C++ code
* these ONNX graphs were designed to be driven by) or confirmed directly against the bundled {@code .onnx}
* files via {@code OrtSession.getInputInfo()}/{@code getOutputInfo()} — not guessed.
*
* <p>{@code detectFaces} always runs both models: a face with no embedding would be useless to
* {@code FaceClusteringService}, so there is no "detect only" mode to expose.
*/
@Component
public class YuNetSFaceFaceDetectionEngine implements IFaceDetectionEngine, AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(YuNetSFaceFaceDetectionEngine.class);
private static final String SOURCE_PROVIDER = "local-onnx";
// --- YuNet: fixed 640x640 input (confirmed via OrtSession.getInputInfo()), 3 strides. ---
private static final int YUNET_INPUT_SIZE = 640;
private static final int[] YUNET_STRIDES = { 8, 16, 32 };
/** opencv_zoo's own demo.py defaults — kept identical rather than re-tuned without reference photos to validate against. */
private static final double YUNET_SCORE_THRESHOLD = 0.6;
private static final double YUNET_NMS_THRESHOLD = 0.3;
// --- SFace: fixed 112x112 aligned input, 128-d output "fc1". ---
private static final int SFACE_CROP_SIZE = 112;
/**
* SFace's own reference layout (right eye, left eye, nose tip, right mouth corner, left mouth corner) in
* 112×112 space — quoted verbatim from OpenCV's {@code face_recognize.cpp}. YuNet's 5 landmarks decode in
* this exact same order, so no reordering is needed between the two models.
*/
private static final double[][] SFACE_REFERENCE_LANDMARKS = {
{ 38.2946, 51.6963 }, { 73.5318, 51.5014 }, { 56.0252, 71.7366 }, { 41.5493, 92.3655 }, { 70.7299, 92.2041 }
};
private final OrtEnvironment environment = OrtEnvironment.getEnvironment();
private final OrtSession yunetSession;
private final OrtSession sfaceSession;
public YuNetSFaceFaceDetectionEngine() {
this.yunetSession = OnnxModelLoader.load("/models/recognition/face_detection_yunet_2023mar.onnx");
this.sfaceSession = OnnxModelLoader.load("/models/recognition/face_recognition_sface_2021dec.onnx");
}
@Override
public List<DetectedRegion> detectFaces(BufferedImage image) {
try {
List<RawFace> faces = detectRaw(image);
List<DetectedRegion> regions = new ArrayList<>(faces.size());
for (RawFace face : faces) {
float[] embedding = embed(image, face.landmarks());
double boxWidth = face.x2() - face.x1();
double boxHeight = face.y2() - face.y1();
BoundingBox box = new BoundingBox((face.x1() + boxWidth / 2) / image.getWidth(),
(face.y1() + boxHeight / 2) / image.getHeight(),
boxWidth / image.getWidth(), boxHeight / image.getHeight());
regions.add(new DetectedRegion(EEntityKind.PERSON, box, face.score(), embedding, null, SOURCE_PROVIDER));
}
return regions;
}
catch (OrtException e) {
log.warn("Face detection/embedding failed unexpectedly: {}", e.toString(), e);
return List.of();
}
}
private record RawFace(double x1, double y1, double x2, double y2, double score, double[] landmarks) {
}
/** YuNet inference + per-stride decode + NMS, boxes/landmarks already scaled back to {@code image}'s own pixel space. */
private List<RawFace> detectRaw(BufferedImage image) throws OrtException {
BufferedImage squashed = resize(image, YUNET_INPUT_SIZE, YUNET_INPUT_SIZE);
float[] chw = OnnxImageTensors.toChwFloats(squashed, true); // YuNet expects BGR (no swapRB in OpenCV's blobFromImage call)
List<RawFace> candidates = new ArrayList<>();
try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, YUNET_INPUT_SIZE, YUNET_INPUT_SIZE });
OrtSession.Result result = yunetSession.run(Map.of("input", input))) {
for (int stride : YUNET_STRIDES) {
decodeStride(result, stride, candidates);
}
}
List<NonMaxSuppression.Box> boxes = new ArrayList<>(candidates.size());
for (int i = 0; i < candidates.size(); i++) {
RawFace c = candidates.get(i);
boxes.add(new NonMaxSuppression.Box(c.x1(), c.y1(), c.x2(), c.y2(), c.score(), i));
}
List<Integer> kept = NonMaxSuppression.suppress(boxes, YUNET_NMS_THRESHOLD);
double scaleX = image.getWidth() / (double) YUNET_INPUT_SIZE;
double scaleY = image.getHeight() / (double) YUNET_INPUT_SIZE;
List<RawFace> scaled = new ArrayList<>(kept.size());
for (int index : kept) {
RawFace c = candidates.get(index);
double[] landmarks = new double[10];
for (int i = 0; i < 5; i++) {
landmarks[2 * i] = c.landmarks()[2 * i] * scaleX;
landmarks[2 * i + 1] = c.landmarks()[2 * i + 1] * scaleY;
}
scaled.add(new RawFace(c.x1() * scaleX, c.y1() * scaleY, c.x2() * scaleX, c.y2() * scaleY, c.score(), landmarks));
}
return scaled;
}
@SuppressWarnings("unchecked")
private void decodeStride(OrtSession.Result result, int stride, List<RawFace> out) throws OrtException {
float[][][] cls = (float[][][]) result.get("cls_" + stride).orElseThrow().getValue();
float[][][] obj = (float[][][]) result.get("obj_" + stride).orElseThrow().getValue();
float[][][] bbox = (float[][][]) result.get("bbox_" + stride).orElseThrow().getValue();
float[][][] kps = (float[][][]) result.get("kps_" + stride).orElseThrow().getValue();
int side = YUNET_INPUT_SIZE / stride;
int count = cls[0].length;
for (int idx = 0; idx < count; idx++) {
int r = idx / side;
int c = idx % side;
double clsScore = clamp01(cls[0][idx][0]);
double objScore = clamp01(obj[0][idx][0]);
double score = Math.sqrt(clsScore * objScore);
if (score < YUNET_SCORE_THRESHOLD) {
continue;
}
float[] bb = bbox[0][idx];
double cx = (c + bb[0]) * stride;
double cy = (r + bb[1]) * stride;
double w = Math.exp(bb[2]) * stride;
double h = Math.exp(bb[3]) * stride;
float[] kp = kps[0][idx];
double[] landmarks = new double[10];
for (int n = 0; n < 5; n++) {
landmarks[2 * n] = (kp[2 * n] + c) * stride;
landmarks[2 * n + 1] = (kp[2 * n + 1] + r) * stride;
}
out.add(new RawFace(cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2, score, landmarks));
}
}
/** Aligns {@code image} onto SFace's reference layout using {@code landmarks} (5 points, original image coordinates), then embeds. */
private float[] embed(BufferedImage image, double[] landmarks) throws OrtException {
double[][] src = new double[5][2];
for (int i = 0; i < 5; i++) {
src[i][0] = landmarks[2 * i];
src[i][1] = landmarks[2 * i + 1];
}
double[] t = SimilarityTransform.estimate(src, SFACE_REFERENCE_LANDMARKS);
BufferedImage aligned = new BufferedImage(SFACE_CROP_SIZE, SFACE_CROP_SIZE, BufferedImage.TYPE_INT_RGB);
Graphics2D g = aligned.createGraphics();
try {
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
// dst = a*src + t, as an AffineTransform: x' = aRe*x - aIm*y + tRe ; y' = aIm*x + aRe*y + tIm
g.drawImage(image, new AffineTransform(t[0], t[1], -t[1], t[0], t[2], t[3]), null);
}
finally {
g.dispose();
}
float[] chw = OnnxImageTensors.toChwFloats(aligned, false); // SFace swaps to RGB internally (swapRB=true)
try (OnnxTensor input = OnnxTensor.createTensor(environment, FloatBuffer.wrap(chw), new long[]{ 1, 3, SFACE_CROP_SIZE, SFACE_CROP_SIZE });
OrtSession.Result result = sfaceSession.run(Map.of("data", input))) {
float[][] embedding = (float[][]) result.get("fc1").orElseThrow().getValue();
return l2Normalize(embedding[0]);
}
}
private static float[] l2Normalize(float[] vector) {
double norm = 0;
for (float v : vector) {
norm += v * v;
}
norm = Math.sqrt(norm);
if (norm == 0) {
return vector;
}
float[] normalized = new float[vector.length];
for (int i = 0; i < vector.length; i++) {
normalized[i] = (float) (vector[i] / norm);
}
return normalized;
}
private static double clamp01(double value) {
return Math.max(0, Math.min(1, value));
}
private static BufferedImage resize(BufferedImage source, int width, int height) {
BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
Graphics2D g = resized.createGraphics();
try {
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
g.drawImage(source, 0, 0, width, height, null);
}
finally {
g.dispose();
}
return resized;
}
@Override
@PreDestroy
public void close() throws OrtException {
yunetSession.close();
sfaceSession.close();
}
}
@@ -22,6 +22,7 @@ import org.icroco.pholio.infra.library.MediaAnalysisService;
import org.icroco.pholio.infra.library.MediaFileService;
import org.icroco.pholio.infra.library.MediaMetadataEditService;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.infra.task.TaskType;
import org.icroco.pholio.ui.common.Disposable;
@@ -139,7 +140,8 @@ public class GalleryView extends HBox implements Disposable, SelectionSource {
MediaMetadataEditService metadataEditService,
IPlaceSearchService placeSearchService,
MediaAnalysisService mediaAnalysisService,
GallerySearchState gallerySearchState) {
GallerySearchState gallerySearchState,
FaceRegionQueryService faceRegionQueryService) {
this.libraryFolderService = libraryFolderService;
this.preferences = preferences;
this.fullImageCache = fullImageCache;
@@ -155,7 +157,7 @@ public class GalleryView extends HBox implements Disposable, SelectionSource {
galleryPane = new ThumbnailGalleryPane(mediaFileService, taskService, mediaLibraryState, preferences, i18n, imageCache);
underConstructionPane = new UnderConstructionPane();
detailPane = new PhotoDetailPane(fullImageCache);
mediaInfoPane = new MediaInfoPane(i18n);
mediaInfoPane = new MediaInfoPane(i18n, faceRegionQueryService);
galleryPane.setOnOpenRequest((file, sourceThumbnail) -> openDetail(file, sourceThumbnail, null));
galleryPane.setOnRegenerateThumbnails(this::regenerateThumbnails);
galleryPane.setOnSetGpsForFiles(this::openLocationDialogForFiles);
@@ -17,6 +17,7 @@ import javafx.scene.control.Label;
import javafx.scene.control.ScrollPane;
import javafx.scene.input.Clipboard;
import javafx.scene.input.ClipboardContent;
import javafx.scene.layout.FlowPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Priority;
import javafx.scene.layout.StackPane;
@@ -27,6 +28,8 @@ import javafx.util.Duration;
import org.icroco.pholio.domain.library.MediaFile;
import org.icroco.pholio.domain.media.GeoLocation;
import org.icroco.pholio.domain.media.MediaMetadata;
import org.icroco.pholio.domain.recognition.EEntityKind;
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.ui.common.Disposable;
import org.icroco.pholio.ui.control.StarRatingControl;
import org.icroco.pholio.ui.i18n.I18nService;
@@ -83,7 +86,8 @@ public class MediaInfoPane extends StackPane implements Disposable {
/** How long {@link #setOpen} takes to grow/shrink this pane's own width. */
private static final Duration SLIDE_DURATION = Duration.millis(240);
private final I18nService i18n;
private final I18nService i18n;
private final FaceRegionQueryService faceRegionQueryService;
private final Label title = new Label();
private final Button closeButton = new Button();
@@ -129,8 +133,9 @@ public class MediaInfoPane extends StackPane implements Disposable {
private Consumer<MediaFile> onEditLocation = file -> {};
private BiConsumer<MediaFile, Integer> onEditRating = (file, rating) -> {};
public MediaInfoPane(I18nService i18n) {
public MediaInfoPane(I18nService i18n, FaceRegionQueryService faceRegionQueryService) {
this.i18n = i18n;
this.faceRegionQueryService = faceRegionQueryService;
getStyleClass().add("media-info-pane");
// Starts fully collapsed — GalleryView seeds the real open/closed state right after construction.
@@ -253,6 +258,7 @@ public class MediaInfoPane extends StackPane implements Disposable {
nodes.add(ratingRow(file, metadata));
nodes.add(dateRow(file, metadata));
cameraRow(metadata).ifPresent(nodes::add);
personsRow(file).ifPresent(nodes::add);
nodes.add(fileRow(file, metadata));
nodes.add(locationRow(file, metadata));
Optional<GeoLocation> location = metadata.geoLocation();
@@ -333,6 +339,35 @@ public class MediaInfoPane extends StackPane implements Disposable {
return Optional.of(iconRow(Feather.CAMERA, camera.orElse(null), specs.isEmpty() ? null : String.join(" ", specs)));
}
/**
* Every person/animal {@code FaceRegionQueryService} has on file for {@code file}, one chip each — a
* named person shows their name, an unnamed cluster (or the future confirmation panel not having run
* yet) shows {@code "gallery.info.unknownPerson"}, and an animal shows its detected species. Read-only:
* naming/confirming a region is the future person-management panel's job, not this pane's.
*/
private Optional<Node> personsRow(MediaFile file) {
Long mediaFileId = file.id();
if (mediaFileId == null) {
return Optional.empty();
}
List<FaceRegionQueryService.DisplayRegion> regions = faceRegionQueryService.findDisplayRegionsFor(mediaFileId);
if (regions.isEmpty()) {
return Optional.empty();
}
FlowPane chips = new FlowPane(6, 6);
regions.forEach(region -> chips.getChildren().add(personChip(region)));
return Optional.of(chips);
}
private Label personChip(FaceRegionQueryService.DisplayRegion region) {
String text = region.kind() == EEntityKind.PERSON
? region.personName() != null ? region.personName() : i18n.get("gallery.info.unknownPerson")
: region.label() != null ? region.label() : i18n.get("gallery.info.unknownPerson");
Label chip = new Label(text);
chip.getStyleClass().add("media-info-person-chip");
return chip;
}
private static String shutterLabel(double seconds) {
return seconds >= 1
? String.format(Locale.ROOT, "%.1fs", seconds)
+9
View File
@@ -371,6 +371,15 @@
-fx-padding: 1 4 1 4;
}
/* One detected person/animal on MediaInfoPane's persons row — a pill, same rounding convention as
.media-info-edit-button, so an unnamed cluster ("Unknown person") reads as a real value, not a link. */
.media-info-person-chip {
-fx-background-color: -color-bg-inset;
-fx-background-radius: 999px;
-fx-padding: 3 10 3 10;
-fx-font-size: 12px;
}
/*
* Transient outcome messages, top-right. The layer itself paints nothing: it is a click-through overlay,
* and each toast is an AtlantaFX Notification carrying its own surface.
@@ -0,0 +1,41 @@
-- See V1's header comment: identifiers are quoted so H2 keeps them lower snake_case, matching what
-- Spring Data JDBC generates.
-- One row per recognized identity. Only PERSON rows are linked to from media_face_region in this
-- iteration — an ANIMAL region never gets one (species lives directly on media_face_region.label, no
-- per-animal identity yet).
CREATE TABLE "person"
(
"id" BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"kind" VARCHAR(16) NOT NULL,
"name" VARCHAR(255),
"created_at" TIMESTAMP NOT NULL,
CONSTRAINT "ck_person_kind" CHECK ("kind" IN ('PERSON', 'ANIMAL'))
);
-- One row per detected face/animal bounding box. person_id stays NULL until FaceClusteringService (PERSON)
-- links it; ANIMAL rows never get one. Re-detection replaces a file's rows wholesale (delete-then-reinsert),
-- the same convention media_file_tag already uses for its own re-scans.
CREATE TABLE "media_face_region"
(
"id" BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"media_file_id" BIGINT NOT NULL REFERENCES "media_file" ("id") ON DELETE CASCADE,
"person_id" BIGINT REFERENCES "person" ("id") ON DELETE SET NULL,
"kind" VARCHAR(16) NOT NULL,
-- Normalized MWG-RS "stArea": x/y is the region's CENTER, w/h its size, fractions of the full image
-- (0..1) — identical numbers to what an mwg-rs:Area XMP struct stores, zero conversion on round-trip.
"area_x" DOUBLE NOT NULL,
"area_y" DOUBLE NOT NULL,
"area_w" DOUBLE NOT NULL,
"area_h" DOUBLE NOT NULL,
"confidence" DOUBLE NOT NULL,
"embedding" VARBINARY(8192),
"label" VARCHAR(255),
"source_provider" VARCHAR(255) NOT NULL,
"confirmed" BOOLEAN NOT NULL DEFAULT FALSE,
"detected_at" TIMESTAMP NOT NULL,
CONSTRAINT "ck_media_face_region_kind" CHECK ("kind" IN ('PERSON', 'ANIMAL'))
);
CREATE INDEX "ix_media_face_region_media_file" ON "media_face_region" ("media_file_id");
CREATE INDEX "ix_media_face_region_person" ON "media_face_region" ("person_id");
+5
View File
@@ -102,6 +102,7 @@ gallery.unknownDate=Unknown date
gallery.info.hash=Hash
gallery.info.noMetadata=No metadata available
gallery.info.addPlace=Add a location
gallery.info.unknownPerson=Unknown person
gallery.location.edit.title=Add a location
gallery.location.edit.searchPrompt=Search for a place…
gallery.location.edit.hint=Changes to the place a photo was taken are saved to the library and, when the format supports it, to the file itself.
@@ -170,6 +171,10 @@ settings.geocoding.cities5000LastImport=Cities5000 reference data last imported
settings.geocoding.cities5000RowCount=Cities5000 reference row count
settings.geocoding.providersJson=Address search providers
settings.geocoding.activeProvider=Active address search provider
settings.recognition.enabled=Face/animal recognition enabled
settings.recognition.providersJson=Recognition providers
settings.recognition.activeProvider=Active recognition provider
settings.recognition.clusterThreshold=Person clustering similarity threshold
settings.ai.provider=Provider
settings.ai.endpoint=Endpoint
settings.imports.largeFolderThreshold=Confirm above (files)
@@ -104,6 +104,7 @@ gallery.unknownDate=Date inconnue
gallery.info.hash=Hash
gallery.info.noMetadata=Aucune métadonnée disponible
gallery.info.addPlace=Ajouter un lieu
gallery.info.unknownPerson=Personne inconnue
gallery.location.edit.title=Ajouter un lieu
gallery.location.edit.searchPrompt=Rechercher un lieu…
gallery.location.edit.hint=Les modifications apportées au lieu de prise de vue seront enregistrées dans la photothèque et, si le format le permet, dans le fichier lui-même.
@@ -172,6 +173,10 @@ settings.geocoding.cities5000LastImport=Dernière importation des données de r
settings.geocoding.cities5000RowCount=Nombre de lignes de référence Cities5000
settings.geocoding.providersJson=Fournisseurs de recherche d'adresse
settings.geocoding.activeProvider=Fournisseur de recherche d'adresse actif
settings.recognition.enabled=Reconnaissance de visages/animaux activée
settings.recognition.providersJson=Fournisseurs de reconnaissance
settings.recognition.activeProvider=Fournisseur de reconnaissance actif
settings.recognition.clusterThreshold=Seuil de similarité pour le regroupement de personnes
settings.ai.provider=Fournisseur
settings.ai.endpoint=Point d'accès
settings.imports.largeFolderThreshold=Confirmer au-delà de (fichiers)
@@ -0,0 +1,22 @@
# Bundled recognition models
Third-party ONNX weights bundled with Pholio for local face/animal recognition. None were modified —
each is used exactly as published upstream.
## face_detection_yunet_2023mar.onnx
- Source: [opencv/opencv_zoo](https://github.com/opencv/opencv_zoo/tree/main/models/face_detection_yunet)
- License: MIT
- Used by: `YuNetSFaceFaceDetectionEngine` (face bounding box + 5-point landmarks)
## face_recognition_sface_2021dec.onnx
- Source: [opencv/opencv_zoo](https://github.com/opencv/opencv_zoo/tree/main/models/face_recognition_sface)
- License: Apache-2.0
- Used by: `YuNetSFaceFaceDetectionEngine` (128-d face embedding, for `FaceClusteringService`)
## yolox_nano.onnx
- Source: [Megvii-BaseDetection/YOLOX](https://github.com/Megvii-BaseDetection/YOLOX), release `0.1.1rc0`
- License: Apache-2.0
- Used by: `YoloXAnimalDetectionEngine` (COCO 80-class detector, filtered to animal classes 14-23)
Binary file not shown.
+36
View File
@@ -281,6 +281,42 @@ geocoding:
visible: false
editable: false
# Local/remote face+animal recognition — same shape as `geocoding` above, edited only through Maintenance's
# recognition tab, never through this generic settings form.
recognition:
enabled:
type: BOOLEAN
label: settings.recognition.enabled
default-value: true
visible: false
editable: false
# Edited only through Maintenance's recognition tab — a JSON-encoded list of RecognitionProviderConfig.
providers-json:
type: STRING
label: settings.recognition.providersJson
default-value: "[]"
visible: false
editable: false
# The active RecognitionProviderConfig's name, or blank to mean "local only" — see RecognitionService.
active-provider:
type: STRING
label: settings.recognition.activeProvider
default-value: ""
visible: false
editable: false
# Cosine-similarity threshold above which FaceClusteringService links an unnamed face to an existing
# Person cluster rather than minting a new one. 0.363 is SFace's own calibrated same-identity threshold
# (OpenCV Zoo's face_recognition_sface README) — the embedding is SFace's, so its own threshold applies.
person-cluster-threshold:
type: DOUBLE
label: settings.recognition.clusterThreshold
default-value: 0.363
visible: false
editable: false
# Onboarding, shown once on the first run. Not offered in the settings view: there is nothing to configure,
# only a fact to remember once the coach-mark sequence has been dismissed or completed.
onboarding:
@@ -0,0 +1,43 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
class NonMaxSuppressionTest {
@Test
void suppressesTheLowerScoredOfTwoHeavilyOverlappingBoxes() {
List<NonMaxSuppression.Box> boxes = List.of(
new NonMaxSuppression.Box(0, 0, 10, 10, 0.9, 0),
new NonMaxSuppression.Box(1, 1, 11, 11, 0.5, 1));
List<Integer> kept = NonMaxSuppression.suppress(boxes, 0.3);
assertThat(kept).containsExactly(0);
}
@Test
void keepsBothOfTwoNonOverlappingBoxes() {
List<NonMaxSuppression.Box> boxes = List.of(
new NonMaxSuppression.Box(0, 0, 10, 10, 0.9, 0),
new NonMaxSuppression.Box(100, 100, 110, 110, 0.5, 1));
List<Integer> kept = NonMaxSuppression.suppress(boxes, 0.3);
assertThat(kept).containsExactlyInAnyOrder(0, 1);
}
@Test
void ordersSurvivorsHighestScoreFirst() {
List<NonMaxSuppression.Box> boxes = List.of(
new NonMaxSuppression.Box(0, 0, 10, 10, 0.4, 0),
new NonMaxSuppression.Box(100, 100, 110, 110, 0.9, 1));
List<Integer> kept = NonMaxSuppression.suppress(boxes, 0.3);
assertThat(kept).containsExactly(1, 0);
}
}
@@ -0,0 +1,58 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import org.assertj.core.api.SoftAssertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.within;
class SimilarityTransformTest {
@Test
void recoversAnExactUniformScaleAndTranslation() {
double[][] src = { { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } };
double[][] dst = new double[src.length][2];
for (int i = 0; i < src.length; i++) {
dst[i][0] = 2 * src[i][0] + 10;
dst[i][1] = 2 * src[i][1] + 5;
}
double[] t = SimilarityTransform.estimate(src, dst);
SoftAssertions.assertSoftly(softly -> {
softly.assertThat(t[0]).as("scale (real part)").isCloseTo(2.0, within(1e-9));
softly.assertThat(t[1]).as("rotation (imaginary part)").isCloseTo(0.0, within(1e-9));
softly.assertThat(t[2]).as("translation x").isCloseTo(10.0, within(1e-9));
softly.assertThat(t[3]).as("translation y").isCloseTo(5.0, within(1e-9));
});
}
@Test
void recoversAnExactRotation() {
// 90 degrees counter-clockwise about the origin: (x, y) -> (-y, x)
double[][] src = { { 1, 0 }, { 0, 1 }, { -1, 0 }, { 0, -1 } };
double[][] dst = { { 0, 1 }, { -1, 0 }, { 0, -1 }, { 1, 0 } };
double[] t = SimilarityTransform.estimate(src, dst);
SoftAssertions.assertSoftly(softly -> {
softly.assertThat(t[0]).as("scale (real part)").isCloseTo(0.0, within(1e-9));
softly.assertThat(t[1]).as("rotation (imaginary part)").isCloseTo(1.0, within(1e-9));
softly.assertThat(t[2]).as("translation x").isCloseTo(0.0, within(1e-9));
softly.assertThat(t[3]).as("translation y").isCloseTo(0.0, within(1e-9));
});
}
@Test
void identityWhenSourceAlreadyMatchesDestination() {
double[][] points = { { 3, 4 }, { 5, 1 }, { -2, 7 } };
double[] t = SimilarityTransform.estimate(points, points);
SoftAssertions.assertSoftly(softly -> {
softly.assertThat(t[0]).isCloseTo(1.0, within(1e-9));
softly.assertThat(t[1]).isCloseTo(0.0, within(1e-9));
softly.assertThat(t[2]).isCloseTo(0.0, within(1e-9));
softly.assertThat(t[3]).isCloseTo(0.0, within(1e-9));
});
}
}
@@ -0,0 +1,37 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import ai.onnxruntime.OrtException;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import org.junit.jupiter.api.Test;
import java.awt.image.BufferedImage;
import java.util.List;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
/** Smoke test — see {@link YuNetSFaceFaceDetectionEngineTest}'s own javadoc for why this isn't an accuracy test. */
class YoloXAnimalDetectionEngineTest {
@Test
void loadsBundledModelAndRunsInferenceWithoutThrowing() throws OrtException {
try (YoloXAnimalDetectionEngine engine = new YoloXAnimalDetectionEngine()) {
BufferedImage image = randomImage(640, 480);
List<DetectedRegion> regions = engine.detectAnimals(image);
assertThat(regions).isNotNull();
}
}
private static BufferedImage randomImage(int width, int height) {
BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
Random random = new Random(42);
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
image.setRGB(x, y, random.nextInt(0xFFFFFF));
}
}
return image;
}
}
@@ -0,0 +1,43 @@
package org.icroco.pholio.infra.recognition.engine.onnx;
import ai.onnxruntime.OrtException;
import org.icroco.pholio.domain.recognition.DetectedRegion;
import org.junit.jupiter.api.Test;
import java.awt.image.BufferedImage;
import java.util.List;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
/**
* A smoke test, not an accuracy test: no real face photo ships with this repo (nothing to license/attribute,
* nothing that could be mistaken for real personal data), so this only pins down that the bundled ONNX
* models load and run inference without throwing on this exact JDK/ONNX Runtime combination — the actual
* risk flagged in the recognition feature's implementation plan. A synthetic noise image legitimately finds
* zero faces; that is the correct, expected answer, not a test gap.
*/
class YuNetSFaceFaceDetectionEngineTest {
@Test
void loadsBundledModelsAndRunsInferenceWithoutThrowing() throws OrtException {
try (YuNetSFaceFaceDetectionEngine engine = new YuNetSFaceFaceDetectionEngine()) {
BufferedImage image = randomImage(640, 480);
List<DetectedRegion> regions = engine.detectFaces(image);
assertThat(regions).isNotNull();
}
}
private static BufferedImage randomImage(int width, int height) {
BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
Random random = new Random(42);
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
image.setRGB(x, y, random.nextInt(0xFFFFFF));
}
}
return image;
}
}
@@ -62,7 +62,7 @@ class MediaLibraryStateTest {
boolean readyBefore = state.thumbnailReadyProperty(1L).get();
boolean otherFileReady = state.thumbnailReadyProperty(2L).get();
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(readyBefore).isFalse();
@@ -97,7 +97,7 @@ class MediaLibraryStateTest {
void folderRemovedClearsEveryThumbnailReadyProperty() throws InterruptedException {
FxTestToolkit.requireToolkit();
MediaLibraryState state = new MediaLibraryState();
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(state.thumbnailReadyProperty(1L).get()).isTrue();
@@ -136,7 +136,7 @@ class MediaLibraryStateTest {
void libraryChangedClearsEveryThumbnailReadyProperty() throws InterruptedException {
FxTestToolkit.requireToolkit();
MediaLibraryState state = new MediaLibraryState();
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(state.thumbnailReadyProperty(1L).get()).isTrue();
@@ -13,6 +13,7 @@ import org.icroco.pholio.infra.library.MediaFileService;
import org.icroco.pholio.infra.library.MediaMetadataEditService;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.icroco.pholio.infra.preferences.PreferencesFixture;
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.ui.FxTestToolkit;
import org.icroco.pholio.ui.ViewSwitcher;
@@ -345,7 +346,8 @@ class NavigationRailTest {
mock(MediaMetadataEditService.class),
mock(IPlaceSearchService.class),
mock(MediaAnalysisService.class),
new GallerySearchState());
new GallerySearchState(),
mock(FaceRegionQueryService.class));
when(context.getBean(GalleryView.class)).thenReturn(galleryView);
when(context.getBean(ModulePlaceholderView.class)).thenReturn(new ModulePlaceholderView(i18n));
return new ViewSwitcher(context, new ViewportSelection());
@@ -22,6 +22,7 @@ import org.icroco.pholio.infra.library.MediaFileService;
import org.icroco.pholio.infra.library.MediaMetadataEditService;
import org.icroco.pholio.infra.preferences.AppPreferences;
import org.icroco.pholio.infra.preferences.PreferencesFixture;
import org.icroco.pholio.infra.recognition.FaceRegionQueryService;
import org.icroco.pholio.infra.task.TaskService;
import org.icroco.pholio.ui.FxTestToolkit;
import org.icroco.pholio.ui.i18n.I18nService;
@@ -87,7 +88,8 @@ class GalleryViewTest {
mock(MediaMetadataEditService.class),
mock(IPlaceSearchService.class),
mock(MediaAnalysisService.class),
new GallerySearchState()));
new GallerySearchState(),
mock(FaceRegionQueryService.class)));
}
@Test
@@ -214,7 +216,7 @@ class GalleryViewTest {
new ThumbnailImageCache(localPreferences, taskService),
new FullImageCache(localPreferences, taskService), mock(LibraryFolderService.class),
mock(ModalService.class), mock(MediaMetadataEditService.class), mock(IPlaceSearchService.class),
mock(MediaAnalysisService.class), new GallerySearchState()));
mock(MediaAnalysisService.class), new GallerySearchState(), mock(FaceRegionQueryService.class)));
Stage farStage = onFxThread(() -> {
farView.resize(400, 500);
@@ -518,7 +518,7 @@ class ThumbnailGalleryPaneTest {
clearInvocations(mediaFileService);
List<GalleryRow> rowsBefore = pane.rows().getItems();
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
assertThat(pane.rows().getItems()).as("same instance: no relayout, only the one cell's slot changed")
@@ -565,7 +565,7 @@ class ThumbnailGalleryPaneTest {
Files.createDirectories(thumbnail.getParent());
Files.write(thumbnail, new byte[]{1, 2, 3});
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L)));
runOnFxThread(() -> state.onMediaFileAnalyzed(new MediaFileAnalyzedEvent(1L, false)));
awaitFlush();
Node cardAfter = onFxThread(() -> {