diff --git a/index.js b/index.js
index f26ee30b..3c39b30a 100644
--- a/index.js
+++ b/index.js
@@ -9,6 +9,7 @@ const { randomUUID } = require('crypto')
 
 const validate = require('./schema-validator')
 const Serializer = require('./serializer')
+const RefResolver = require('./ref-resolver')
 const buildAjv = require('./ajv')
 
 let largeArraySize = 2e4
@@ -57,20 +58,12 @@ function resolveRef (location, ref) {
   const schemaId = ref.slice(0, hashIndex) || location.schemaId
   const jsonPointer = ref.slice(hashIndex) || '#'
 
-  const schemaRef = schemaId + jsonPointer
+  const schema = refResolver.getSchema(schemaId, jsonPointer)
 
-  let ajvSchema
-  try {
-    ajvSchema = ajvInstance.getSchema(schemaRef)
-  } catch (error) {
+  if (schema === undefined) {
     throw new Error(`Cannot find reference "${ref}"`)
   }
 
-  if (ajvSchema === undefined) {
-    throw new Error(`Cannot find reference "${ref}"`)
-  }
-
-  const schema = ajvSchema.schema
   if (schema.$ref !== undefined) {
     return resolveRef({ schema, schemaId, jsonPointer }, schema.$ref)
   }
@@ -83,6 +76,7 @@ const objectReferenceSerializersMap = new Map()
 
 let rootSchemaId = null
 let ajvInstance = null
+let refResolver = null
 let contextFunctions = null
 
 function build (schema, options) {
@@ -95,11 +89,13 @@ function build (schema, options) {
   options = options || {}
 
   ajvInstance = buildAjv(options.ajv)
+  refResolver = new RefResolver()
   rootSchemaId = schema.$id || randomUUID()
 
   isValidSchema(schema)
   extendDateTimeType(schema)
   ajvInstance.addSchema(schema, rootSchemaId)
+  refResolver.addSchema(schema, rootSchemaId)
 
   if (options.schema) {
     const externalSchemas = clone(options.schema)
@@ -114,7 +110,14 @@ function build (schema, options) {
         schemaKey = key + externalSchema.$id // relative URI
       }
 
-      if (ajvInstance.getSchema(schemaKey) === undefined) {
+      if (refResolver.getSchema(schemaKey) === undefined) {
+        refResolver.addSchema(externalSchema, key)
+      }
+
+      if (
+        ajvInstance.refs[schemaKey] === undefined &&
+        ajvInstance.schemas[schemaKey] === undefined
+      ) {
         ajvInstance.addSchema(externalSchema, schemaKey)
       }
     }
@@ -178,6 +181,7 @@ function build (schema, options) {
   const stringifyFunc = contextFunc(ajvInstance, serializer)
 
   ajvInstance = null
+  refResolver = null
   rootSchemaId = null
   contextFunctions = null
   arrayItemsReferenceSerializersMap.clear()
@@ -493,6 +497,7 @@ function mergeAllOfSchema (location, schema, mergedSchema) {
 
   mergedSchema.$id = `merged_${randomUUID()}`
   ajvInstance.addSchema(mergedSchema)
+  refResolver.addSchema(mergedSchema)
   location.schemaId = mergedSchema.$id
   location.jsonPointer = '#'
 }
diff --git a/package.json b/package.json
index 57e8453a..94d23fbc 100644
--- a/package.json
+++ b/package.json
@@ -52,6 +52,7 @@
     "@fastify/deepmerge": "^1.0.0",
     "ajv": "^8.10.0",
     "ajv-formats": "^2.1.1",
+    "fast-deep-equal": "^3.1.3",
     "fast-uri": "^2.1.0",
     "rfdc": "^1.2.0"
   },
diff --git a/ref-resolver.js b/ref-resolver.js
new file mode 100644
index 00000000..65f1eeb8
--- /dev/null
+++ b/ref-resolver.js
@@ -0,0 +1,82 @@
+'use strict'
+
+const deepEqual = require('fast-deep-equal')
+
+class RefResolver {
+  constructor () {
+    this.schemas = {}
+  }
+
+  addSchema (schema, schemaId) {
+    if (schema.$id !== undefined && schema.$id.charAt(0) !== '#') {
+      schemaId = schema.$id
+    }
+    this.insertSchemaBySchemaId(schema, schemaId)
+    this.insertSchemaSubschemas(schema, schemaId)
+  }
+
+  getSchema (schemaId, jsonPointer = '#') {
+    const schema = this.schemas[schemaId]
+    if (schema === undefined) {
+      return undefined
+    }
+    if (schema.anchors[jsonPointer] !== undefined) {
+      return schema.anchors[jsonPointer]
+    }
+    return getDataByJSONPointer(schema.schema, jsonPointer)
+  }
+
+  insertSchemaBySchemaId (schema, schemaId) {
+    if (
+      this.schemas[schemaId] !== undefined &&
+      !deepEqual(schema, this.schemas[schemaId].schema)
+    ) {
+      throw new Error(`There is already another schema with id ${schemaId}`)
+    }
+    this.schemas[schemaId] = { schema, anchors: {} }
+  }
+
+  insertSchemaByAnchor (schema, schemaId, anchor) {
+    const { anchors } = this.schemas[schemaId]
+    if (
+      anchors[anchor] !== undefined &&
+      !deepEqual(schema, anchors[anchor])
+    ) {
+      throw new Error(`There is already another schema with id ${schemaId}#${anchor}`)
+    }
+    anchors[anchor] = schema
+  }
+
+  insertSchemaSubschemas (schema, rootSchemaId) {
+    const schemaId = schema.$id
+    if (schemaId !== undefined && typeof schemaId === 'string') {
+      if (schemaId.charAt(0) === '#') {
+        this.insertSchemaByAnchor(schema, rootSchemaId, schemaId)
+      } else {
+        this.insertSchemaBySchemaId(schema, schemaId)
+        rootSchemaId = schemaId
+      }
+    }
+
+    for (const key in schema) {
+      if (typeof schema[key] === 'object' && schema[key] !== null) {
+        this.insertSchemaSubschemas(schema[key], rootSchemaId)
+      }
+    }
+  }
+}
+
+function getDataByJSONPointer (data, jsonPointer) {
+  const parts = jsonPointer.split('/')
+  let current = data
+  for (const part of parts) {
+    if (part === '' || part === '#') continue
+    if (typeof current !== 'object' || current === null) {
+      return undefined
+    }
+    current = current[part]
+  }
+  return current
+}
+
+module.exports = RefResolver
diff --git a/test/ref.test.js b/test/ref.test.js
index 3158b6aa..97641370 100644
--- a/test/ref.test.js
+++ b/test/ref.test.js
@@ -1953,3 +1953,31 @@ test('should resolve absolute $refs', (t) => {
 
   t.equal(output, JSON.stringify(object))
 })
+
+test('nested schema should overwrite anchor scope', (t) => {
+  t.plan(2)
+
+  const externalSchema = {
+    root: {
+      $id: 'root',
+      definitions: {
+        subschema: {
+          $id: 'subschema',
+          definitions: {
+            anchorSchema: {
+              $id: '#anchor',
+              type: 'string'
+            }
+          }
+        }
+      }
+    }
+  }
+
+  const data = 'test'
+  const stringify = build({ $ref: 'subschema#anchor' }, { schema: externalSchema })
+  const output = stringify(data)
+
+  t.equal(output, JSON.stringify(data))
+  t.throws(() => build({ $ref: 'root#anchor' }, { schema: externalSchema }))
+})