diff --git a/baseline.xml b/baseline.xml
index 5df827be..a1fae711 100644
--- a/baseline.xml
+++ b/baseline.xml
@@ -29,11 +29,6 @@
       <code><![CDATA[$rawData]]></code>
     </MixedAssignment>
   </file>
-  <file src="src/Metadata/AttributeMetadataFactory.php">
-    <PossiblyNullReference>
-      <code><![CDATA[guess]]></code>
-    </PossiblyNullReference>
-  </file>
   <file src="src/Metadata/ClassMetadata.php">
     <InvalidPropertyAssignmentValue>
       <code><![CDATA[new ReflectionClass($data['className'])]]></code>
@@ -48,6 +43,12 @@
     <InvalidOperand>
       <code><![CDATA[$guessers]]></code>
     </InvalidOperand>
+    <InvalidReturnStatement>
+      <code><![CDATA[$this->classMetadata[$class]]]></code>
+    </InvalidReturnStatement>
+    <InvalidReturnType>
+      <code><![CDATA[ClassMetadata<T>]]></code>
+    </InvalidReturnType>
     <MixedReturnStatement>
       <code><![CDATA[(new ReflectionClass($class))->newLazyProxy(
             function () use ($metadata, $data): object {
diff --git a/src/Metadata/AttributeMetadataFactory.php b/src/Metadata/AttributeMetadataFactory.php
index b4a2eb69..def68bfb 100644
--- a/src/Metadata/AttributeMetadataFactory.php
+++ b/src/Metadata/AttributeMetadataFactory.php
@@ -36,12 +36,9 @@
 
 final class AttributeMetadataFactory implements MetadataFactory
 {
-    /** @var array<class-string, ClassMetadata> */
-    private array $classMetadata = [];
-
     private readonly TypeResolver $typeResolver;
 
-    private readonly Guesser|null $guesser;
+    private readonly Guesser $guesser;
 
     public function __construct(
         TypeResolver|null $typeResolver = null,
@@ -60,13 +57,6 @@ public function __construct(
      */
     public function metadata(string $class): ClassMetadata
     {
-        if (array_key_exists($class, $this->classMetadata)) {
-            /** @var ClassMetadata<T> $classMetadata */
-            $classMetadata = $this->classMetadata[$class];
-
-            return $classMetadata;
-        }
-
         try {
             $reflectionClass = new ReflectionClass($class);
         } catch (ReflectionException) {
@@ -89,15 +79,6 @@ public function metadata(string $class): ClassMetadata
      */
     private function getClassMetadata(ReflectionClass $reflectionClass): ClassMetadata
     {
-        $class = $reflectionClass->getName();
-
-        if (array_key_exists($class, $this->classMetadata)) {
-            /** @var ClassMetadata<T> $classMetadata */
-            $classMetadata = $this->classMetadata[$class];
-
-            return $classMetadata;
-        }
-
         $metadata = new ClassMetadata(
             $reflectionClass,
             $this->getPropertyMetadataList($reflectionClass),
@@ -115,8 +96,6 @@ private function getClassMetadata(ReflectionClass $reflectionClass): ClassMetada
             );
         }
 
-        $this->classMetadata[$class] = $metadata;
-
         return $metadata;
     }
 
diff --git a/src/MetadataHydrator.php b/src/MetadataHydrator.php
index 961741e4..0352669c 100644
--- a/src/MetadataHydrator.php
+++ b/src/MetadataHydrator.php
@@ -35,6 +35,9 @@ final class MetadataHydrator implements Hydrator
     /** @var array<int, class-string> */
     private array $stack = [];
 
+    /** @var array<class-string, ClassMetadata> */
+    private array $classMetadata = [];
+
     public function __construct(
         private readonly MetadataFactory $metadataFactory = new AttributeMetadataFactory(),
         PayloadCryptographer|null $cryptographer = null,
@@ -65,7 +68,7 @@ public function __construct(
     public function hydrate(string $class, array $data): object
     {
         try {
-            $metadata = $this->metadataFactory->metadata($class);
+            $metadata = $this->metadata($class);
         } catch (ClassNotFound $e) {
             throw new ClassNotSupported($class, $e);
         }
@@ -132,10 +135,6 @@ private function doHydrate(ClassMetadata $metadata, array $data): object
             $normalizer = $propertyMetadata->normalizer;
 
             if ($normalizer) {
-                if ($normalizer instanceof HydratorAwareNormalizer) {
-                    $normalizer->setHydrator($this);
-                }
-
                 try {
                     /** @psalm-suppress MixedAssignment */
                     $value = $normalizer->denormalize($value);
@@ -182,7 +181,7 @@ public function extract(object $object): array
         $this->stack[$objectId] = $object::class;
 
         try {
-            $metadata = $this->metadataFactory->metadata($object::class);
+            $metadata = $this->metadata($object::class);
 
             foreach ($metadata->preExtractCallbacks as $callback) {
                 $callback->invoke($object);
@@ -197,10 +196,6 @@ public function extract(object $object): array
                 $normalizer = $propertyMetadata->normalizer;
 
                 if ($normalizer) {
-                    if ($normalizer instanceof HydratorAwareNormalizer) {
-                        $normalizer->setHydrator($this);
-                    }
-
                     try {
                         /** @psalm-suppress MixedAssignment */
                         $value = $normalizer->normalize($value);
@@ -261,6 +256,32 @@ private function promotedConstructorParametersWithDefaultValue(ClassMetadata $me
         return $result;
     }
 
+    /**
+     * @param class-string<T> $class
+     *
+     * @return ClassMetadata<T>
+     *
+     * @template T of object
+     */
+    private function metadata(string $class): ClassMetadata
+    {
+        if (array_key_exists($class, $this->classMetadata)) {
+            return $this->classMetadata[$class];
+        }
+
+        $this->classMetadata[$class] = $metadata = $this->metadataFactory->metadata($class);
+
+        foreach ($metadata->properties as $property) {
+            if (!($property->normalizer instanceof HydratorAwareNormalizer)) {
+                continue;
+            }
+
+            $property->normalizer->setHydrator($this);
+        }
+
+        return $metadata;
+    }
+
     /** @param iterable<Guesser> $guessers */
     public static function create(
         iterable $guessers = [],
diff --git a/tests/Unit/Metadata/AttributeMetadataFactoryTest.php b/tests/Unit/Metadata/AttributeMetadataFactoryTest.php
index bb133bbb..74f02a93 100644
--- a/tests/Unit/Metadata/AttributeMetadataFactoryTest.php
+++ b/tests/Unit/Metadata/AttributeMetadataFactoryTest.php
@@ -52,16 +52,6 @@ public function testEmptyObject(): void
         self::assertCount(0, $metadata->postHydrateCallbacks);
     }
 
-    public function testSameMetadata(): void
-    {
-        $object = new class {
-        };
-
-        $metadataFactory = new AttributeMetadataFactory();
-
-        self::assertSame($metadataFactory->metadata($object::class), $metadataFactory->metadata($object::class));
-    }
-
     public function testNotFoundProperty(): void
     {
         $this->expectException(PropertyMetadataNotFound::class);