diff --git a/pkg/agent/secondarynetwork/podwatch/controller.go b/pkg/agent/secondarynetwork/podwatch/controller.go
index b9ba8052bf6..c325fe54472 100644
--- a/pkg/agent/secondarynetwork/podwatch/controller.go
+++ b/pkg/agent/secondarynetwork/podwatch/controller.go
@@ -144,9 +144,20 @@ func (pc *PodController) buildVFDeviceIDListPerPod(podName, podNamespace string)
 		vfDeviceIDInfoCache = append(vfDeviceIDInfoCache, initSriovVfDeviceID)
 	}
 	pc.vfDeviceIDUsageMap.Store(podKey, vfDeviceIDInfoCache)
+	klog.V(2).InfoS("Pod specific SRIOV VF cache created", "Key", podKey)
 	return vfDeviceIDInfoCache, nil
 }
 
+func (pc *PodController) deleteVFDeviceIDListPerPod(podName, podNamespace string) {
+	podKey := podNamespace + "/" + podName
+	_, cacheFound := pc.vfDeviceIDUsageMap.Load(podKey)
+	if cacheFound {
+		pc.vfDeviceIDUsageMap.Delete(podKey)
+		klog.V(2).InfoS("Pod specific SRIOV VF cache cleared", "Key", podKey)
+	}
+	return
+}
+
 func (pc *PodController) assignUnusedSriovVFDeviceIDPerPod(podName, podNamespace, interfaceName string) (string, error) {
 	var cache []podSriovVFDeviceIDInfo
 	cache, err := pc.buildVFDeviceIDListPerPod(podName, podNamespace)
@@ -290,6 +301,8 @@ func (pc *PodController) handleRemovePod(key string) error {
 		} else {
 			// Delete cache entry from podCNIInfo.
 			pc.podCache.DeleteCNIConfigInfo(containerInfo)
+			// Delete Pod specific VF cache (if one exists)
+			pc.deleteVFDeviceIDListPerPod(containerInfo.PodName, containerInfo.PodNameSpace)
 		}
 	}
 	return nil
diff --git a/pkg/agent/secondarynetwork/podwatch/controller_test.go b/pkg/agent/secondarynetwork/podwatch/controller_test.go
index a0d8b8bd23e..2c157117ecd 100644
--- a/pkg/agent/secondarynetwork/podwatch/controller_test.go
+++ b/pkg/agent/secondarynetwork/podwatch/controller_test.go
@@ -510,4 +510,23 @@ func TestPodControllerAddPod(t *testing.T) {
 		// we don't expect an error here, no requeueing
 		assert.NoError(t, podController.handleAddUpdatePod(pod))
 	})
+
+	t.Run("Error when adding VF deviceID cache per Pod", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+		network := testNetwork(networkName)
+		podController, _, _ := newPodController(ctrl)
+		podController.podCache.AddCNIConfigInfo(cniConfig)
+		_, err := podController.kubeClient.CoreV1().Pods(testNamespace).Create(context.Background(), pod, metav1.CreateOptions{})
+		require.NoError(t, err, "error when creating test Pod")
+		_, err = podController.netAttachDefClient.NetworkAttachmentDefinitions(testNamespace).Create(context.Background(), network, metav1.CreateOptions{})
+		require.NoError(t, err, "error when creating test NetworkAttachmentDefinition")
+
+		_, err = podController.assignUnusedSriovVFDeviceIDPerPod(podName, testNamespace, interfaceName)
+		require.NoError(t, err, "error while assigning unused VfDevice ID")
+
+		podController.deleteVFDeviceIDListPerPod(podName, testNamespace)
+		require.NoError(t, err, "error deleting cache")
+
+	})
 }